1
2 """
3 12_to_13.py - migration from < moin--main--patch-248 to >= patch 249
4 * convert event-log from iso8859-1 to config.charset (utf-8) encoding
5
6 Steps for a successful migration to utf-8:
7 1. stop your wiki and make a backup
8 2. make a copy of the wiki's "data" directory to your working dir
9 3. clean up your working copy of the data dir:
10 a. if you use CVS or GNU arch remove stuff like CVS/, .cvsignore
11 or .arch-ids/ etc.
12 b. remove *.pickle (used by moin for caching some information,
13 will be re-created automatically), especially:
14 I. data/user/userdict.pickle
15 II. data/dicts.pickle
16 c. if you used symlinks in data/text or elsewhere, remove them
17 4. make sure that from_encoding and to_encoding matches your needs (see
18 beginning of script below and config.charset in moin_config.py) and
19 run python2.3 12_to_13_mig6.py from your working dir
20 5. if there was no error, you will find:
21 data.pre-mig6 (the script renames your data directory copy to that name)
22 data (result, converted to utf-8)
23 6. verify conversion results (number of pages, size of logs, attachments,
24 number of backup copies) - everything should be reasonable before
25 you proceed. Usually the file size gets larger when converting from
26 iso8859-1 (or other non-unicode charset) to utf-8 except if your
27 content is ASCII-only, then it will keep its size.
28 7. copy additional files from data.pre-mig6 to data (maybe intermaps, logs,
29 etc.). Be aware that the file contents AND file names of wiki content
30 may have changed, so DO NOT copy the cache/ directory, but let
31 the wiki recreate it.
32 8. replace the data directory your wiki uses with the data directory
33 you created by previous steps. DO NOT simply copy the converted stuff
34 into the original or you will duplicate pages and create chaos!
35 9. test it. if something has gone wrong, you still have your backup.
36
37
38 10. if you use dictionaries for spellchecking, you have to convert them
39 to config.charset, too. Remove your dict.cache before re-starting
40 your wiki.
41
42 @copyright: 2004 Thomas Waldmann
43 @license: GPL, see COPYING for details
44 """
45
46
47 from_encoding = 'utf-8'
48 to_encoding = 'utf-8'
49
50 import os.path, sys, shutil, urllib
51
52 sys.path.insert(0, '../../..')
53 from MoinMoin import wikiutil
54
55 from migutil import opj, listdir, copy_file, copy_dir
56
57 errorcount = 0
58
59 def convert_string(str, enc_from, enc_to):
60
61 global errorcount
62 try:
63 return str.decode(enc_from).encode(enc_to)
64 except:
65
66 errorcount +=1
67 return "ERROR"
68
69
70 def convert_eventlog(fname_from, fname_to, enc_from, enc_to):
71 print "%s -> %s" % (fname_from, fname_to)
72 file_from = open(fname_from)
73 file_to = open(fname_to, "w")
74
75 for line in file_from:
76 line = line.replace('\r','')
77 line = line.replace('\n','')
78 fields = line.split('\t')
79 kvpairs = fields[2]
80 kvpairs = kvpairs.split('&')
81 kvlist = []
82 for kvpair in kvpairs:
83 key, val = kvpair.split('=')
84 key = urllib.unquote(key)
85 val = urllib.unquote(val)
86 key = convert_string(key, enc_from, enc_to)
87 val = convert_string(val, enc_from, enc_to)
88 if("ERROR"==key or "ERROR"==val):
89
90
91 pass
92 else:
93 key = urllib.quote(key)
94 val = urllib.quote(val)
95 kvlist.append("%s=%s" % (key,val))
96
97
98
99 fields[2] = '&'.join(kvlist)
100 line = '\t'.join(fields) + '\n'
101 file_to.write(line)
102
103 file_to.close()
104 file_from.close()
105 st=os.stat(fname_from)
106 os.utime(fname_to, (st.st_atime,st.st_mtime))
107
108
109 origdir = 'data.pre-mig6'
110
111 try:
112 os.rename('data', origdir)
113 pass
114 except OSError:
115 print "You need to be in the directory where your copy of the 'data' directory is located."
116 sys.exit(1)
117
118 copy_dir(origdir, 'data')
119 os.remove(opj('data','event-log'))
120 convert_eventlog(opj(origdir, 'event-log'), opj('data', 'event-log'), from_encoding, to_encoding)
121
122 print "mig6 finished but maybe there is %d lines log not transition for some unicode ERROR!!"%errorcount
