Browse Source

Optimize translator from smd-client (pipeline mode)

Avoid spawning a process every time a translation has to happen.
Also refactor some code, removing copy/paste
Enrico Tassi 8 years ago
parent
commit
5e2ad7e17e
4 changed files with 108 additions and 79 deletions
  1. 7
    3
      README
  2. 7
    0
      smd-translate
  3. 87
    70
      syncmaildir.lua
  4. 7
    6
      tests.d/pull-push/14-translator

+ 7
- 3
README View File

@@ -297,7 +297,11 @@ like `Maildir/cur` or `Maildir/.sub.folder/new` to local names like
297 297
 could be:
298 298
 
299 299
     #!/bin/sh
300
-    sed 's/^Maildir\(.*\)$/Mail\1/' | sed 's?/\.?/?'
300
+    sed --unbuffered -e 's/^Maildir\(.*\)$/Mail\1/' -e 's?/\.?/?'
301
+
302
+Note the `--unbuffered`: translators should not work in buffered mode.
303
+I.e. when given a line in input (terminated by `\n`) they must output a
304
+line without expecting any additional input.
301 305
 
302 306
 Translating the way back is trickier, since the leading `.` must be
303 307
 added only to sub-folders:
@@ -305,9 +309,9 @@ added only to sub-folders:
305 309
     #!/bin/sh
306 310
     t() {
307 311
         if [ "$1" = Mail/cur -o "$1" = Mail/tmp -o "$1" = Mail/new ]; then
308
-            echo $1 | sed 's?^Mail/\(.*\)?Maildir/\1?'
312
+            echo $1 | sed --unbuffered 's?^Mail/\(.*\)?Maildir/\1?'
309 313
         else
310
-            echo $1 | sed 's?^Mail/\(.*\)?Maildir/.\1?'
314
+            echo $1 | sed --unbuffered 's?^Mail/\(.*\)?Maildir/.\1?'
311 315
         fi
312 316
     }
313 317
     while read M; do t "$M"; done

+ 7
- 0
smd-translate View File

@@ -13,6 +13,13 @@ function escape(x)
13 13
 	return x:gsub('[]%%%.-]',function(x) return '%'..x end)
14 14
 end
15 15
 
16
+local orig_print = print
17
+
18
+function print(...)
19
+	orig_print(...)
20
+	io.stdout:flush()
21
+end
22
+
16 23
 -- argument parsing
17 24
 local mode, dir, endpoint
18 25
 mode = 'default'

+ 87
- 70
syncmaildir.lua View File

@@ -40,6 +40,74 @@ if string.sub(SMDVERSION,1,1) == '@' then
40 40
 		SMDVERSION = '0.0.0'
41 41
 end
42 42
 
43
+-- to call external filter processes without too much pain
44
+function make_slave_filter_process(cmd, seed)
45
+	seed = seed or "no seed"
46
+	local init = function(filter)
47
+		if filter.inf == nil then
48
+			local rc
49
+			local base_dir
50
+			local home = os.getenv('HOME')
51
+			local user = os.getenv('USER') or 'nobody'
52
+			local mangled_name = string.gsub(seed,"[ %./]",'-')
53
+			local attempt = 0
54
+			if home ~= nil then
55
+				base_dir = home ..'/.smd/fifo/'
56
+			else
57
+				base_dir = '/tmp/'
58
+			end
59
+			rc = os.execute(MDDIFF..' --mkdir-p '..quote(base_dir))
60
+			if rc ~= 0 then
61
+				log_internal_error_and_fail('unable to create directory',
62
+					"make_slave_filter_process")
63
+			end
64
+			repeat 
65
+				pipe = base_dir..'smd-'..user..os.time()..mangled_name..attempt
66
+				attempt = attempt + 1
67
+				rc = os.execute(MDDIFF..' --mkfifo '..quote(pipe))
68
+			until rc == 0 or attempt > 10
69
+			if rc ~= 0 then
70
+				log_internal_error_and_fail('unable to create fifo',
71
+					"make_slave_filter_process")
72
+			end
73
+			filter.inf = io.popen(cmd(quote(pipe)),'r')
74
+			filter.outf = io.open(pipe,'w')
75
+			filter.pipe = pipe
76
+		end
77
+	end
78
+	return setmetatable({}, {
79
+		__index = {
80
+			read = function(filter,...)
81
+				if filter.inf == nil then
82
+					-- check already initialized
83
+					log_internal_error_and_fail("read called before write",
84
+						"make_slave_filter_process")
85
+				end
86
+				-- once we known the channel is open, we clean up the fifo
87
+				if not filter.removed and filter.did_write then
88
+					filter.removed = true
89
+					local rc = { filter.inf:read(...) }
90
+					os.remove(filter.pipe)
91
+					return unpack(rc)
92
+				else
93
+					return filter.inf:read(...)
94
+				end
95
+			end,
96
+			write = function(filter,...)
97
+				init(filter)
98
+				filter.did_write = true
99
+				return filter.outf:write(...)
100
+			end,
101
+			flush = function(filter)
102
+				return filter.outf:flush()
103
+			end,
104
+			lines = function(filter)
105
+				return filter.inf:lines()
106
+			end
107
+		}
108
+	})
109
+end
110
+
43 111
 -- you should use logs_tags_and_fail
44 112
 function error(msg)
45 113
 	local d = debug.getinfo(1,"nl")
@@ -69,17 +137,20 @@ end
69 137
 function dry_run() return dryrun end
70 138
 
71 139
 function set_translator(p)
140
+	local translator_filter = make_slave_filter_process(function(pipe)
141
+		return p .. ' < ' .. pipe
142
+	end, "translate")
72 143
 	if p == 'cat' then translator = function(x) return x end
73 144
 	else translator = function(x)
74
-		local f = io.popen('echo '..quote(x)..' | '..p)
75
-		local rc = f:read('*l')
145
+		translator_filter:write(x..'\n')
146
+		translator_filter:flush()
147
+		local rc = translator_filter:read('*l')
76 148
 		if rc == nil or rc == 'ERROR' then
77 149
 			log_error("Translator "..p.." on input "..x.." gave an error")
78
-			for l in f:lines() do log_error(l) end
150
+			for l in translator_filter:lines() do log_error(l) end
79 151
 			log_tags_and_fail('Unable to translate mailbox',
80 152
 				'translate','bad-translator',true)
81 153
 		end
82
-		f:close()
83 154
 		if rc:match('%.%.') then
84 155
 			log_error("Translator "..p.." on input "..x..
85 156
 				" returned a path containing ..")
@@ -295,46 +366,19 @@ end
295 366
 
296 367
 -- =================== fast/maildir aware mkdir -p ==========================
297 368
 
298
-local mddiff_mkdirln_handler = {}
369
+local mddiff_mkdirln_handler = make_slave_filter_process(function(pipe)
370
+	return MDDIFF .. ' -s ' .. pipe	
371
+end, "mk_link_wa")
299 372
 
300 373
 -- create a link from the workarea to the real mailbox using mddiff
301 374
 function mk_link_wa(src, target)
302
-	local pipe, rm_pipe = nil, false
303
-	if mddiff_mkdirln_handler.inf == nil then
304
-		local rc
305
-		local base_dir
306
-		local home = os.getenv('HOME')
307
-		local user = os.getenv('USER') or 'nobody'
308
-		local mangled_name = string.gsub(src..target,"/","-"):gsub(' ','-')
309
-		local attempt = 0
310
-		if home ~= nil then
311
-			base_dir = home ..'/.smd/fifo/'
312
-		else
313
-			base_dir = '/tmp/'
314
-		end
315
-		repeat 
316
-			pipe = base_dir..'smd-'..user..os.time()..mangled_name..attempt
317
-			attempt = attempt + 1
318
-			mkdir_p(pipe)
319
-			rc = os.execute(MDDIFF..' --mkfifo '..quote(pipe))
320
-		until rc == 0 or attempt > 10
321
-		if rc ~= 0 then
322
-			log_internal_error_and_fail('unable to create fifo', "mk_link_wa")
323
-		end
324
-		mddiff_mkdirln_handler.inf = io.popen(MDDIFF .. ' -s ' .. quote(pipe))
325
-		mddiff_mkdirln_handler.outf = io.open(pipe,'w')
326
-		rm_pipe = true
327
-	end
328
-	mddiff_mkdirln_handler.outf:write(src,'\n',target,'\n')
329
-	mddiff_mkdirln_handler.outf:flush()
330
-	local data = mddiff_mkdirln_handler.inf:read('*l')
375
+	mddiff_mkdirln_handler:write(src,'\n',target,'\n')
376
+	mddiff_mkdirln_handler:flush()
377
+	local data = mddiff_mkdirln_handler:read('*l')
331 378
 	if data:match('^ERROR') or not data:match('^OK') then
332 379
 		log_tags_and_fail('Failed to mddiff -s',
333 380
 			'mddiff-s','wrong-permissions',true)
334 381
 	end
335
-	-- we are now sure that both endpoints opened the file, thus
336
-	-- we can safely garbage collect it
337
-	if rm_pipe then os.remove(pipe) end
338 382
 end
339 383
 
340 384
 local mkdir_p_cache = {}
@@ -488,38 +532,14 @@ function parse(s,spec)
488 532
 	return unpack(res)
489 533
 end
490 534
 
491
-local mddiff_sha_handler = {}
535
+local mddiff_sha_handler = make_slave_filter_process(function(pipe)
536
+	return MDDIFF .. ' ' .. pipe
537
+end, "sha_file")
492 538
 
493 539
 function sha_file(name)
494
-	local pipe, rm_pipe = nil, false
495
-	if mddiff_sha_handler.inf == nil then
496
-		local rc
497
-		local base_dir
498
-		local home = os.getenv('HOME')
499
-		local user = os.getenv('USER') or 'nobody'
500
-		local mangled_name = string.gsub(name,"/","-"):gsub(' ','-')
501
-		local attempt = 0
502
-		if home ~= nil then
503
-			base_dir = home ..'/.smd/fifo/'
504
-		else
505
-			base_dir = '/tmp/'
506
-		end
507
-		repeat 
508
-			pipe = base_dir..'smd-'..user..os.time()..mangled_name..attempt
509
-			attempt = attempt + 1
510
-			mkdir_p(pipe)
511
-			rc = os.execute(MDDIFF..' --mkfifo '..quote(pipe))
512
-		until rc == 0 or attempt > 10
513
-		if rc ~= 0 then
514
-			log_internal_error_and_fail('unable to create fifo', "sha_file")
515
-		end
516
-		mddiff_sha_handler.inf = io.popen(MDDIFF .. ' ' .. quote(pipe))
517
-		mddiff_sha_handler.outf = io.open(pipe,'w')
518
-		rm_pipe = true
519
-	end
520
-	mddiff_sha_handler.outf:write(name..'\n')
521
-	mddiff_sha_handler.outf:flush()
522
-	local data = mddiff_sha_handler.inf:read('*l')
540
+	mddiff_sha_handler:write(name,'\n')
541
+	mddiff_sha_handler:flush()
542
+	local data = mddiff_sha_handler:read('*l')
523 543
 	if data:match('^ERROR') then
524 544
 		log_tags_and_fail("Failed to sha1 message: "..(name or "nil"),
525 545
 			'sha_file','modify-while-update',false,'retry')
@@ -528,9 +548,6 @@ function sha_file(name)
528 548
 	if hsha == nil or bsha == nil then
529 549
 		log_internal_error_and_fail('mddiff incorrect behaviour', "mddiff")
530 550
 	end
531
-	-- we are now sure that both endpoints opened the file, thus
532
-	-- we can safely garbage collect it
533
-	if rm_pipe then os.remove(pipe) end
534 551
 	return hsha, bsha
535 552
 end
536 553
 

+ 7
- 6
tests.d/pull-push/14-translator View File

@@ -8,14 +8,15 @@ echo MAILBOX_LOCAL=MyMail >> target/.smd/config.default
8 8
 echo MAILBOX_REMOTE=Mail >> target/.smd/config.default
9 9
 echo TRANSLATOR_RL=../trl >> target/.smd/config.default
10 10
 echo TRANSLATOR_LR=../tlr >> target/.smd/config.default
11
+echo DEBUG=true >> target/.smd/config.default
11 12
 
12 13
 cat > tlr <<EOT
13 14
 #!/bin/sh
14 15
 t() {
15 16
 if [ "\$1" = MyMail/cur -o "\$1" = MyMail/tmp -o "\$1" = MyMail/new ]; then
16
-	echo \$1 | sed 's?^MyMail/\(.*\)?Mail/\1?'
17
+	echo \$1 | sed --unbuffered 's?^MyMail/\(.*\)?Mail/\1?'
17 18
 else
18
-	echo \$1 | sed 's?^MyMail/\(.*\)?Mail/.\1?'
19
+	echo \$1 | sed --unbuffered 's?^MyMail/\(.*\)?Mail/.\1?'
19 20
 fi
20 21
 }
21 22
 while read M; do t "\$M"; done
@@ -23,7 +24,7 @@ EOT
23 24
 chmod a+x tlr
24 25
 cat > trl <<EOT
25 26
 #!/bin/sh
26
-sed 's/^Mail\(.*\)$/MyMail\1/' | sed 's?/\.?/?'
27
+sed --unbuffered -e 's/^Mail\(.*\)$/MyMail\1/' -e 's?/\.?/?'
27 28
 EOT
28 29
 chmod a+x trl
29 30
 
@@ -31,7 +32,7 @@ mkdir -p Mail/.foo.bar
31 32
 cp -r Mail/cur Mail/.foo.bar/
32 33
 
33 34
 mpull
34
-assert $? 0 "failed mpull"
35
+assert $? 0 "failed mpull 1"
35 36
 test_eq Mail/cur/ target/MyMail/cur/
36 37
 test_eq Mail/.foo.bar/ target/MyMail/foo.bar/
37 38
 
@@ -40,14 +41,14 @@ cp Mail/.foo.bar/cur/`ls Mail/.foo.bar/cur/|head -n 1` Mail/.foo.bar/cur/here
40 41
 sed -i s/a/z/ Mail/.foo.bar/cur/here
41 42
 
42 43
 mpull
43
-assert $? 0 "failed mpull"
44
+assert $? 0 "failed mpull 2"
44 45
 test_eq Mail/.foo.bar/ target/MyMail/foo.bar/
45 46
 
46 47
 mkdir -p Mail/.foo.baz/cur
47 48
 cp Mail/.foo.bar/cur/here Mail/.foo.baz/cur/here
48 49
 
49 50
 mpull
50
-assert $? 0 "failed mpull"
51
+assert $? 0 "failed mpull 3"
51 52
 test_eq Mail/.foo.bar/ target/MyMail/foo.bar/
52 53
 
53 54
 touch target/.smd/workarea/cruft

Loading…
Cancel
Save