Commit d16f9272 authored by Mario Manno's avatar Mario Manno

pentabarf import updated

* more documentation on importing from pentabarf
* improved event rating import speed
* carefully import long links
* fix user role import
* redirect deleted users
* respect conference timezone on event import
* sorted Gemfile
parent 99f6b506
......@@ -4,34 +4,34 @@ ruby "1.9.3"
gem 'rails', '3.2.11'
gem 'sqlite3'
gem 'pg'
gem 'mysql2'
gem 'pg'
gem 'sqlite3'
gem 'acts_as_commentable'
gem 'acts_as_indexed'
gem 'barista'
gem 'bcrypt-ruby'
gem 'cancan'
gem 'haml'
gem 'acts_as_commentable'
gem 'will_paginate'
gem 'paperclip', '< 3.0'
gem 'gravatar-ultimate'
gem 'cocoon'
gem 'formtastic', '~> 2.1.0'
gem 'formtastic-bootstrap', :git => "git://github.com/katastrophie/formtastic-bootstrap.git"
gem 'gravatar-ultimate'
gem 'haml'
gem 'jquery-rails', '~> 1.0.19'
gem 'acts_as_indexed'
gem 'cocoon'
gem 'paper_trail', '2.3.3'
gem 'json'
gem 'localized_language_select', '0.2.0', :git => "git://github.com/oneiros/localized_language_select.git"
gem 'nokogiri'
gem 'paperclip', '< 3.0'
gem 'paper_trail', '2.3.3'
gem 'prawn'
gem 'prawn_rails'
gem 'ransack'
gem 'transitions', :require => ["transitions", "active_record/transitions"]
gem 'json'
gem 'barista'
gem 'ri_cal'
gem 'nokogiri'
gem 'settingslogic'
gem 'transitions', :require => ["transitions", "active_record/transitions"]
gem 'twitter-bootstrap-rails', :git => "git://github.com/seyhunak/twitter-bootstrap-rails.git", :ref => "5e62b21c8f258010af7f5bc858b89a24f16936a9"
gem 'formtastic-bootstrap', :git => "git://github.com/katastrophie/formtastic-bootstrap.git"
gem 'prawn'
gem 'prawn_rails'
gem 'will_paginate'
group :development, :test do
gem 'bullet'
......@@ -40,19 +40,19 @@ group :development, :test do
end
group :test do
gem 'minitest'
gem 'factory_girl_rails', '~> 1.2.0'
gem 'minitest'
gem 'turn', :require => false
end
group :development do
gem 'hpricot'
gem 'yaml_db'
gem 'shotgun'
gem 'yaml_db'
end
group :assets do
gem 'sass-rails', " ~> 3.2.0"
gem 'coffee-rails', " ~> 3.2.0"
gem 'sass-rails', " ~> 3.2.0"
gem 'uglifier'
end
......@@ -5,13 +5,15 @@ These notes may help to import data from a pentabarf postgresql database.
Using postgresql as a database for frab is still somewhat untested. The pentabarf
import is however likely to fail, as pentabarf uses text fields instead of char(255)
Imagemagick needs to be installed as we will convert pjpeg and tiff to png.
## postgresql installation
install postgresql
Install postgresql
## postgresql setup
* Make it listen on localhost
* make it listen on localhost
* create a psql user and grant some access on relations
* add a pentabarf entry for the postgresql database to your rails db environment
......@@ -27,13 +29,22 @@ Make a copy of your postgresql database, as we need to do some changes
Grant all permissions on the database copy to the import user account:
psql cccv
select 'grant all on '||schemaname||'.'||tablename||' to bar;' from pg_tables
psql NEWNAME
-- generate the grant statements
select 'grant all on '||schemaname||'.'||tablename||' to frab;' from pg_tables
order by schemaname, tablename;
-- copy&paste the generated statements into psql
-- in case you re-created the NEWNAME copy, re-grant permissions to the user
REVOKE ALL ON SCHEMA public FROM frab;
GRANT ALL ON SCHEMA public TO frab;
REVOKE ALL ON SCHEMA auth FROM frab;
GRANT ALL ON SCHEMA auth TO frab;
## data migration
I had to delete some images, too.
Conference acronyms need to be within /^[a-zA-Z0-9_-]*$/
Whitespaces are removed automatically, but you need to replace unicode characters manually.
-- conference acronyms appear in URLs, they may not contain whitespace in frab:
UPDATE conference SET acronym = replace(acronym, ' ', '');
......@@ -41,7 +52,6 @@ I had to delete some images, too.
## import
Delete any old mappings from previous imports. Maybe delete the old filess, too.
rm tmp/*mappings.yml
......@@ -49,4 +59,19 @@ Delete any old mappings from previous imports. Maybe delete the old filess, too.
RAILS_ENV=production rake db:reset
RAILS_ENV=production rake pentabarf:import:all
## testing
You can check on the barf data like this:
RAILS_ENV="development" rails console
@p = PentabarfImportHelper.new
@barf = @p.instance_variable_get('@barf')
@barf.select_all("SELECT * FROM conference")
## privileges
You maybe want to drop all users to the coordinator role, to start fresh.
User.all.select { |u| u.role == "admin" or u.role == "orga" }.each { |u| puts "dropping ${u.email}"; u.role = "coordinator"; u.save }
......@@ -49,7 +49,13 @@ class ApplicationController < ActionController::Base
end
def current_user
@current_user ||= User.find(session[:user_id]) if session[:user_id]
user = nil
# maybe the user got deleted, so lets wrap this in a rescue block
begin
user = User.find(session[:user_id]) if session[:user_id]
rescue
end
@current_user ||= user
end
def authenticate_user!
......
......@@ -12,11 +12,11 @@ class PentabarfImportHelper
# pentabarf roles are just different
ROLE_MAPPING = {
"submitter" => "submitter",
"reviewer" => "reviewer",
"comittee" => "coordinator",
"developer" => "admin",
"admin" => "orga",
"submitter" => "submitter",
"reviewer" => "reviewer"
"developer" => "admin",
}
EVENT_STATE_MAPPING = {
......@@ -54,19 +54,30 @@ class PentabarfImportHelper
conferences = @barf.select_all("SELECT * FROM conference")
puts "[ ] importing #{conferences.count} conferences" if DEBUG
conference_mapping = create_mappings(:conferences)
conferences.each do |conference|
penta_days = @barf.select_values("SELECT conference_day FROM conference_day
WHERE conference_id = #{conference["conference_id"]}
ORDER BY conference_day ASC")
fake_days = penta_days
if fake_days.empty?
fake_days << Time.now.ago(1.year)
end
new_conference = Conference.create!(
:title => conference["title"],
# clean up acronyms in pentabarf db first!
:acronym => conference["acronym"],
:acronym => conference["acronym"].gsub(/ /,''),
# it's a string like 'Europe/Berlin', 'Berlin'
:timezone => conference["timezone"],
#TODO omg timeslots
:timeslot_duration => interval_to_minutes(conference["timeslot_duration"]),
:default_timeslots => conference["default_timeslots"],
:max_timeslots => conference["max_timeslot_duration"],
:feedback_enabled => conference["f_feedback_enabled"],
:feedback_enabled => conference["f_feedback_enabled"] == 'f' ? false : true,
# nowhere to find. just use the conference dates instead..
:created_at => fake_days.first.to_datetime,
:updated_at => fake_days.last.to_datetime,
# TODO ticket server, instead of link? DO TICKET URLS THEY END UP PUBLIC?
)
conference_mapping[conference["conference_id"]] = new_conference.id
......@@ -76,21 +87,31 @@ class PentabarfImportHelper
#puts "+++ %s - %s" % [conference["acronym"], Time.zone] if DEBUG
# convert pentabarf days to frab days
penta_days = @barf.select_values("SELECT conference_day FROM conference_day
WHERE conference_id = #{conference["conference_id"]}
ORDER BY conference_day ASC")
penta_days.each do |day|
day = day.to_datetime
# pentabarf uses a 'day change time', which is set at 04:00 o'clock for ccc congresses
# pentabarf uses a 'day change time',
# which is set at 04:00 o'clock for ccc congresses
# so we kind of fix it:
start_date = day.to_datetime.change(:hour => 4, :minute => 0)
end_date = day.since(1.days).to_datetime.change(:hour => 3, :minute => 59)
hour = conference['day_change'].gsub(/:.*/,'').to_i
start_date = day.to_datetime.change(:hour => hour, :minute => 0)
end_date = day.since(1.days).to_datetime.change(:hour => hour-1, :minute => 59)
tmp = Day.new(:conference => new_conference,
:start_date => Time.zone.local_to_utc(start_date),
:end_date => Time.zone.local_to_utc(end_date))
tmp.save!
end
# create a dummy cfp for this conference
cfp = CallForPapers.new
cfp.conference = new_conference
cfp.start_date = fake_days.first.to_datetime.ago(3.month)
cfp.end_date = fake_days.first.to_datetime.ago(1.month)
cfp.created_at = cfp.start_date
cfp.updated_at = cfp.end_date
cfp.info_url = conference["homepage"]
cfp.contact_email = conference["email"]
cfp.save!
end
save_mappings(:conferences)
end
......@@ -222,7 +243,7 @@ class PentabarfImportHelper
:password_confirmation => password
)
user.confirmed_at = Time.now
user.role = role ? "submitter" : ROLE_MAPPING[role]
user.role = role ? ROLE_MAPPING[role] : "submitter"
user.pentabarf_salt = account["salt"]
user.pentabarf_password = account["password"]
user.save!
......@@ -248,21 +269,23 @@ class PentabarfImportHelper
def import_links
mappings(:people).each do |orig_id, new_id|
links = @barf.select_all("SELECT l.title, l.url FROM conference_person as p LEFT OUTER JOIN conference_person_link as l ON p.conference_person_id = l.conference_person_id WHERE p.person_id = #{orig_id}")
puts "[ ] importing #{links.count} links from people" if DEBUG
# puts "[ ] importing #{links.count} links from people" if DEBUG
links.each do |link|
if link["title"] and link["url"]
person = Person.find(new_id)
Link.create(:title => link["title"], :url => link["url"], :linkable => person)
Link.create(:title => truncate_string(link["title"]),
:url => truncate_string(link["url"]), :linkable => person)
end
end
end
mappings(:events).each do |orig_id, new_id|
links = @barf.select_all("SELECT title, url FROM event_link WHERE event_id = #{orig_id}")
puts "[ ] importing #{links.count} links from events" if DEBUG
#puts "[ ] importing #{links.count} links from events" if DEBUG
links.each do |link|
if link["title"] and link["url"]
event = Event.find(new_id)
Link.create(:title => link["title"], :url => link["url"], :linkable => event)
Link.create(:title => truncate_string(link["title"]),
:url => truncate_string(link["url"]), :linkable => event)
end
end
end
......@@ -276,6 +299,8 @@ class PentabarfImportHelper
image = @barf.select_one("SELECT * FROM event_image WHERE event_id = #{event["event_id"]}")
image_file = image_to_file(image, "event_id")
conference = Conference.find(mappings(:conferences)[event["conference_id"]])
Time.zone = conference.timezone
new_event = Event.create!(
:conference_id => conference.id,
:track_id => mappings(:tracks)[event["conference_track_id"]],
......@@ -287,7 +312,6 @@ class PentabarfImportHelper
#:state => EVENT_STATE_MAPPING[event["event_state"]],
:state => EVENT_PROGRESS_MAPPING[event["event_state_progress"]],
:language => event["language"],
#TODO
:start_time => start_time(event["conference_day"], event["start_time"]),
:room_id => mappings(:rooms)[event["conference_room_id"]],
:abstract => event["abstract"],
......@@ -304,6 +328,8 @@ class PentabarfImportHelper
end
def import_event_ratings
disable_event_callback(EventRating)
event_ratings = @barf.select_all("SELECT * FROM event_rating")
puts "[ ] importing #{event_ratings.count} event ratings" if DEBUG
event_ratings.each do |rating|
......@@ -314,12 +340,16 @@ class PentabarfImportHelper
:comment => rating["remark"],
)
end
# update in batch
puts "[ ] updating rating average on events" if DEBUG
update_event_average("event_ratings", "average_rating")
enable_event_callbacks(EventRating)
end
def import_event_feedbacks
# we wan't to update this in batch after all the inserts
Event.skip_callback(:save, :after, :update_conflicts)
EventFeedback.skip_callback(:save, :after, :update_average)
disable_event_callback(EventFeedback)
event_feedbacks = @barf.select_all("SELECT * FROM event_feedback")
puts "[ ] importing #{event_feedbacks.count} event feedbacks" if DEBUG
event_feedbacks.each do |feedback|
......@@ -339,7 +369,7 @@ class PentabarfImportHelper
end
# this might happen:
if rating.nil?
rating = -99
rating = 3
end
EventFeedback.create!(
:event_id => mappings(:events)[feedback["event_id"]],
......@@ -348,26 +378,11 @@ class PentabarfImportHelper
:created_at => feedback["eval_time"]
)
end
# update counts in batch
puts "[ ] updating feedbacks counters on events" if DEBUG
# FIXME speed problems: 26h
# Event.joins(:event_feedbacks).readonly(false).all.each {|e| e.recalculate_average_feedback!}
# direct sqlite syntax: 10min?
ActiveRecord::Base.connection.execute "UPDATE events SET average_feedback=(
SELECT sum(rating)/count(rating)
FROM event_feedbacks WHERE events.id = event_feedbacks.event_id)"
# other databases?
# UPDATE events
# SET average_feedback=(sum(rating)/count(rating))
# FROM events
# INNER JOIN event_feedbacks
# ON id = event_id
# re-enable after_save callbacks
Event.set_callback(:save, :after, :update_conflicts)
EventFeedback.set_callback(:save, :after, :update_average)
# update in batch
puts "[ ] updating feedback average on events" if DEBUG
update_event_average("event_feedbacks", "average_feedback")
enable_event_callbacks(EventFeedback)
end
def import_event_attachments
......@@ -455,7 +470,8 @@ class PentabarfImportHelper
def start_time(day, interval)
return nil unless day and interval
Time.parse(day + " " + interval)
t = Time.parse(day + " " + interval)
Time.zone.utc_to_local(t).in_time_zone
end
def image_to_file(image, id_column)
......@@ -522,4 +538,35 @@ class PentabarfImportHelper
def mappings_file(name)
Rails.root.join('tmp', "#{name}_mappings.yml").to_s
end
def update_event_average(table, field)
# FIXME speed problems: 26h
# Event.joins(:event_feedbacks).readonly(false).all.each {|e| e.recalculate_average_feedback!}
# direct sqlite syntax: 10min?
ActiveRecord::Base.connection.execute "UPDATE events SET #{field}=(
SELECT sum(rating)/count(rating)
FROM #{table} WHERE events.id = #{table}.event_id)"
# other databases?
# UPDATE events
# SET average_feedback=(sum(rating)/count(rating))
# FROM events
# INNER JOIN event_feedbacks
# ON id = event_id
end
def disable_event_callback(average)
# we wan't to update this in batch after all the inserts
Event.skip_callback(:save, :after, :update_conflicts)
average.skip_callback(:save, :after, :update_average)
# TODO disable counter_cache?
end
def enable_event_callbacks(average)
# re-enable after_save callbacks
Event.set_callback(:save, :after, :update_conflicts)
average.set_callback(:save, :after, :update_average)
end
end
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment