Synchronize notmuch mail across machines
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

muchsync.cc 12KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506
  1. #include <cstring>
  2. #include <iomanip>
  3. #include <iostream>
  4. #include <sstream>
  5. #include <fcntl.h>
  6. #include <getopt.h>
  7. #include <unistd.h>
  8. #include <sys/stat.h>
  9. #include <sys/types.h>
  10. #include "misc.h"
  11. #include "muchsync.h"
  12. #include "infinibuf.h"
  13. using namespace std;
  14. #if 0
  15. // This gives core dumps to make it easier to debug
  16. struct no_such_exception_t {
  17. const char *what() noexcept { return "no such exception"; }
  18. };
  19. using whattocatch_t = no_such_exception_t;
  20. #else
  21. using whattocatch_t = const exception;
  22. #endif
  23. #define MUCHSYNC_DEFDIR "/.notmuch/muchsync"
  24. const char muchsync_defdir[] = MUCHSYNC_DEFDIR;
  25. const char muchsync_dbpath[] = MUCHSYNC_DEFDIR "/state.db";
  26. const char muchsync_trashdir[] = MUCHSYNC_DEFDIR "/trash";
  27. const char muchsync_tmpdir[] = MUCHSYNC_DEFDIR "/tmp";
  28. constexpr char shell[] = "/bin/sh";
  29. // Probably no win from buffering more than 128MB of input data from net
  30. constexpr size_t max_buf_size = 0x8000000;
  31. bool opt_fullscan;
  32. bool opt_noscan;
  33. bool opt_init;
  34. bool opt_server;
  35. bool opt_upbg;
  36. bool opt_noup;
  37. bool opt_nonew;
  38. bool opt_newid;
  39. i64 opt_newid_value;
  40. int opt_verbose;
  41. int opt_upbg_fd = -1;
  42. string opt_ssh = "ssh -CTaxq";
  43. string opt_remote_muchsync_path = "muchsync";
  44. string opt_notmuch_config;
  45. string opt_init_dest;
  46. static bool
  47. muchsync_init (const string &maildir, bool create = false)
  48. {
  49. string trashbase = maildir + muchsync_trashdir + "/";
  50. if (!access ((maildir + muchsync_tmpdir).c_str(), 0)
  51. && !access ((trashbase + "ff").c_str(), 0))
  52. return true;
  53. if (create && mkdir (maildir.c_str(), 0777) && errno != EEXIST) {
  54. perror (maildir.c_str());
  55. return false;
  56. }
  57. string notmuchdir = maildir + "/.notmuch";
  58. if (create && access (notmuchdir.c_str(), 0) && errno == ENOENT) {
  59. notmuch_database_t *notmuch;
  60. if (!notmuch_database_create (maildir.c_str(), &notmuch))
  61. notmuch_database_destroy (notmuch);
  62. }
  63. string msdir = maildir + muchsync_defdir;
  64. for (string d : {msdir, maildir + muchsync_trashdir,
  65. maildir + muchsync_tmpdir}) {
  66. if (mkdir (d.c_str(), 0777) && errno != EEXIST) {
  67. perror (d.c_str());
  68. return false;
  69. }
  70. }
  71. for (int i = 0; i < 0x100; i++) {
  72. ostringstream os;
  73. os << trashbase << hex << setfill('0') << setw(2) << i;
  74. if (mkdir (os.str().c_str(), 0777) && errno != EEXIST) {
  75. perror (os.str().c_str());
  76. return false;
  77. }
  78. }
  79. return true;
  80. }
  81. static void
  82. tag_stderr(string tag)
  83. {
  84. infinistreambuf *isb =
  85. new infinistreambuf(new infinibuf_mt);
  86. streambuf *err = cerr.rdbuf(isb);
  87. thread t ([=]() {
  88. istream in (isb);
  89. ostream out (err);
  90. string line;
  91. while (getline(in, line))
  92. out << tag << line << endl;
  93. });
  94. t.detach();
  95. cerr.rdbuf(isb);
  96. }
  97. //[[noreturn]]
  98. void
  99. usage (int code = 1)
  100. {
  101. (code ? cerr : cout) << "\
  102. usage: muchsync\n\
  103. muchsync server [server-options]\n\
  104. muchsync --init maildir server [server-options]\n\
  105. \n\
  106. Additional options:\n\
  107. -C file Specify path to notmuch config file\n\
  108. -F Disable optimizations and do full maildir scan\n\
  109. -v Increase verbosity\n\
  110. -r path Specify path to notmuch executable on server\n\
  111. -s ssh-cmd Specify ssh command and arguments\n\
  112. --config file Specify path to notmuch config file (same as -C)\n\
  113. --nonew Do not run notmuch new first\n\
  114. --noup[load] Do not upload changes to server\n\
  115. --upbg Download mail in forground, then upload in background\n\
  116. --self Print local replica identifier and exit\n\
  117. --newid Change local replica identifier and exit\n\
  118. --version Print version number and exit\n\
  119. --help Print usage\n";
  120. exit (code);
  121. }
  122. static void
  123. id_request()
  124. {
  125. unique_ptr<notmuch_db> nmp;
  126. try {
  127. nmp.reset(new notmuch_db (opt_notmuch_config));
  128. } catch (whattocatch_t &e) { cerr << e.what() << '\n'; exit (1); }
  129. notmuch_db &nm = *nmp;
  130. string dbpath = nm.maildir + muchsync_dbpath;
  131. sqlite3 *db = dbopen(dbpath.c_str(), opt_newid);
  132. if (!db)
  133. exit(1);
  134. cleanup _c (sqlite3_close_v2, db);
  135. if (!opt_newid)
  136. cout << getconfig<i64>(db, "self") << '\n';
  137. else {
  138. sqlexec (db, "BEGIN;");
  139. i64 oldid = getconfig<i64>(db, "self");
  140. sqlexec (db, "INSERT OR IGNORE INTO sync_vector (replica, version)"
  141. " VALUES (%lld, 1);", opt_newid_value);
  142. setconfig (db, "self", opt_newid_value);
  143. cout << "changing id from " << oldid << " to " << opt_newid_value << '\n';
  144. sqlexec (db, "COMMIT;");
  145. }
  146. }
  147. static void
  148. server()
  149. {
  150. ifdinfinistream ibin(0, max_buf_size);
  151. cleanup _fixbuf0 ([](streambuf *sb){ cin.rdbuf(sb); },
  152. cin.rdbuf(ibin.rdbuf()));
  153. ofdstream ibout(1, [&ibin](bool blocked) {
  154. ibin.set_max_buf_size(blocked ? 0 : max_buf_size);
  155. });
  156. cleanup _fixbuf1 ([](streambuf *sb){ cout.rdbuf(sb); },
  157. cout.rdbuf(ibout.rdbuf()));
  158. tag_stderr("[SERVER] ");
  159. unique_ptr<notmuch_db> nmp;
  160. try {
  161. nmp.reset(new notmuch_db (opt_notmuch_config));
  162. } catch (whattocatch_t &e) { cerr << e.what() << '\n'; exit (1); }
  163. notmuch_db &nm = *nmp;
  164. string dbpath = nm.maildir + muchsync_dbpath;
  165. if (!opt_nonew)
  166. nm.run_new();
  167. if (!muchsync_init (nm.maildir))
  168. exit (1);
  169. sqlite3 *db = dbopen(dbpath.c_str());
  170. if (!db)
  171. exit(1);
  172. cleanup _c (sqlite3_close_v2, db);
  173. try {
  174. if (!opt_noscan)
  175. sync_local_data(db, nm.maildir);
  176. muchsync_server(db, nm);
  177. }
  178. catch (whattocatch_t &e) {
  179. cerr << e.what() << '\n';
  180. exit(1);
  181. }
  182. }
  183. static void
  184. cmd_iofds (int fds[2], const string &cmd)
  185. {
  186. int ifds[2], ofds[2];
  187. if (pipe (ifds))
  188. throw runtime_error (string ("pipe: ") + strerror (errno));
  189. if (pipe (ofds)) {
  190. close (ifds[0]);
  191. close (ifds[1]);
  192. throw runtime_error (string ("pipe: ") + strerror (errno));
  193. }
  194. pid_t pid = fork();
  195. switch (pid) {
  196. case -1:
  197. close (ifds[0]);
  198. close (ifds[1]);
  199. close (ofds[0]);
  200. close (ofds[1]);
  201. throw runtime_error (string ("fork: ") + strerror (errno));
  202. break;
  203. case 0:
  204. close (ifds[0]);
  205. close (ofds[1]);
  206. if (ofds[0] != 0) {
  207. dup2 (ofds[0], 0);
  208. close (ofds[0]);
  209. }
  210. if (ifds[1] != 1) {
  211. dup2 (ifds[1], 1);
  212. close (ifds[1]);
  213. }
  214. execl (shell, shell, "-c", cmd.c_str(), nullptr);
  215. cerr << shell << ": " << strerror (errno) << '\n';
  216. _exit (1);
  217. break;
  218. default:
  219. close (ifds[1]);
  220. close (ofds[0]);
  221. fcntl (ifds[0], F_SETFD, 1);
  222. fcntl (ofds[1], F_SETFD, 1);
  223. fds[0] = ifds[0];
  224. fds[1] = ofds[1];
  225. break;
  226. }
  227. }
  228. static void
  229. create_config(istream &in, ostream &out, string &maildir)
  230. {
  231. if (!maildir.size() || !maildir.front())
  232. throw runtime_error ("illegal empty maildir path\n");
  233. string line;
  234. out << "conffile\n";
  235. get_response(in, line);
  236. get_response(in, line);
  237. size_t len = stoul(line.substr(4));
  238. if (len <= 0)
  239. throw runtime_error ("server did not return configuration file\n");
  240. string conf;
  241. conf.resize(len);
  242. if (!in.read(&conf.front(), len))
  243. throw runtime_error ("cannot read configuration file from server\n");
  244. int fd = open(opt_notmuch_config.c_str(), O_CREAT|O_TRUNC|O_WRONLY|O_EXCL,
  245. 0666);
  246. if (fd < 0)
  247. throw runtime_error (opt_notmuch_config + ": " + strerror (errno));
  248. write(fd, conf.c_str(), conf.size());
  249. close(fd);
  250. if (maildir[0] != '/') {
  251. const char *p = getenv("PWD");
  252. if (!p)
  253. throw runtime_error ("no PWD in environment\n");
  254. maildir = p + ("/" + maildir);
  255. }
  256. notmuch_db nm (opt_notmuch_config);
  257. nm.set_config ("database.path", maildir.c_str(), nullptr);
  258. }
  259. static void
  260. client(int ac, char **av)
  261. {
  262. unique_ptr<notmuch_db> nmp;
  263. struct stat sb;
  264. int err = stat(opt_notmuch_config.c_str(), &sb);
  265. if (opt_init) {
  266. if (!err) {
  267. cerr << opt_notmuch_config << " should not exist with --init option\n";
  268. exit (1);
  269. }
  270. else if (errno != ENOENT) {
  271. cerr << opt_notmuch_config << ": " << strerror(errno) << '\n';
  272. exit (1);
  273. }
  274. }
  275. else if (err) {
  276. cerr << opt_notmuch_config << ": " << strerror(errno) << '\n';
  277. exit (1);
  278. }
  279. else {
  280. try {
  281. nmp.reset(new notmuch_db (opt_notmuch_config));
  282. } catch (whattocatch_t &e) { cerr << e.what() << '\n'; exit (1); }
  283. }
  284. if (ac == 0) {
  285. if (!nmp)
  286. usage();
  287. if (!muchsync_init(nmp->maildir, true))
  288. exit (1);
  289. if (!opt_nonew)
  290. nmp->run_new();
  291. string dbpath = nmp->maildir + muchsync_dbpath;
  292. sqlite3 *db = dbopen(dbpath.c_str());
  293. if (!db)
  294. exit (1);
  295. cleanup _c (sqlite3_close_v2, db);
  296. sync_local_data (db, nmp->maildir);
  297. exit(0);
  298. }
  299. ostringstream os;
  300. os << opt_ssh << ' ' << av[0] << ' ' << opt_remote_muchsync_path
  301. << " --server";
  302. for (int i = 1; i < ac; i++)
  303. os << ' ' << av[i];
  304. string cmd (os.str());
  305. int fds[2];
  306. cmd_iofds (fds, cmd);
  307. ifdinfinistream in (fds[0], max_buf_size);
  308. ofdstream out (fds[1], [&in](bool blocked){
  309. in.set_max_buf_size(blocked ? 0 : max_buf_size);
  310. });
  311. in.tie (&out);
  312. if (opt_init) {
  313. create_config(in, out, opt_init_dest);
  314. try {
  315. nmp.reset(new notmuch_db (opt_notmuch_config, true));
  316. } catch (whattocatch_t &e) { cerr << e.what() << '\n'; exit (1); }
  317. }
  318. if (!muchsync_init(nmp->maildir, true))
  319. exit(1);
  320. if (!opt_nonew)
  321. nmp->run_new();
  322. string dbpath = nmp->maildir + muchsync_dbpath;
  323. sqlite3 *db = dbopen(dbpath.c_str(), true);
  324. if (!db)
  325. exit (1);
  326. cleanup _c (sqlite3_close_v2, db);
  327. try {
  328. muchsync_client (db, *nmp, in, out);
  329. }
  330. catch (whattocatch_t &e) {
  331. cerr << e.what() << '\n';
  332. exit (1);
  333. }
  334. }
  335. enum opttag {
  336. OPT_VERSION = 0x100,
  337. OPT_SERVER,
  338. OPT_NOSCAN,
  339. OPT_UPBG,
  340. OPT_NOUP,
  341. OPT_HELP,
  342. OPT_NONEW,
  343. OPT_SELF,
  344. OPT_NEWID,
  345. OPT_INIT
  346. };
  347. static const struct option muchsync_options[] = {
  348. { "version", no_argument, nullptr, OPT_VERSION },
  349. { "server", no_argument, nullptr, OPT_SERVER },
  350. { "noscan", no_argument, nullptr, OPT_NOSCAN },
  351. { "upbg", no_argument, nullptr, OPT_UPBG },
  352. { "noup", no_argument, nullptr, OPT_NOUP },
  353. { "noupload", no_argument, nullptr, OPT_NOUP },
  354. { "nonew", no_argument, nullptr, OPT_NONEW },
  355. { "init", required_argument, nullptr, OPT_INIT },
  356. { "self", no_argument, nullptr, OPT_SELF },
  357. { "newid", optional_argument, nullptr, OPT_NEWID },
  358. { "config", required_argument, nullptr, 'C' },
  359. { "help", no_argument, nullptr, OPT_HELP },
  360. { nullptr, 0, nullptr, 0 }
  361. };
  362. int
  363. main(int argc, char **argv)
  364. {
  365. umask (077);
  366. opt_notmuch_config = notmuch_db::default_notmuch_config();
  367. bool opt_self = false;
  368. int opt;
  369. while ((opt = getopt_long(argc, argv, "+C:Fr:s:v",
  370. muchsync_options, nullptr)) != -1)
  371. switch (opt) {
  372. case 0:
  373. break;
  374. case 'C':
  375. opt_notmuch_config = optarg;
  376. break;
  377. case 'F':
  378. opt_fullscan = true;
  379. break;
  380. case 'r':
  381. opt_remote_muchsync_path = optarg;
  382. break;
  383. case 's':
  384. opt_ssh = optarg;
  385. break;
  386. case 'v':
  387. opt_verbose++;
  388. break;
  389. case OPT_VERSION:
  390. cout << PACKAGE_STRING << '\n';
  391. exit (0);
  392. case OPT_SERVER:
  393. opt_server = true;
  394. break;
  395. case OPT_NOSCAN:
  396. opt_noscan = true;
  397. break;
  398. case OPT_UPBG:
  399. opt_upbg = true;
  400. break;
  401. case OPT_NOUP:
  402. opt_noup = true;
  403. break;
  404. case OPT_NONEW:
  405. opt_nonew = true;
  406. break;
  407. case OPT_SELF:
  408. opt_self = true;
  409. break;
  410. case OPT_NEWID:
  411. opt_newid = true;
  412. if (optarg) {
  413. opt_newid_value = std::stoll(optarg, nullptr, 10);
  414. if (opt_newid_value <= 0) {
  415. cerr << "invalid id " << optarg << '\n';
  416. usage();
  417. }
  418. }
  419. else
  420. opt_newid_value = create_random_id();
  421. break;
  422. case OPT_INIT:
  423. opt_init = true;
  424. opt_init_dest = optarg;
  425. break;
  426. case OPT_HELP:
  427. usage(0);
  428. default:
  429. usage();
  430. }
  431. if (opt_self || opt_newid) {
  432. if ((opt_self && opt_newid) || optind != argc
  433. || opt_init || opt_noup || opt_upbg)
  434. usage();
  435. id_request();
  436. }
  437. else if (opt_server) {
  438. if (opt_init || opt_noup || opt_upbg || optind != argc)
  439. usage();
  440. server();
  441. }
  442. else if (opt_upbg) {
  443. int fds[2];
  444. if (pipe(fds)) {
  445. cerr << "pipe: " << strerror(errno) << '\n';
  446. exit (1);
  447. }
  448. fcntl(fds[1], F_SETFD, 1);
  449. if (fork() > 0) {
  450. char c;
  451. close(fds[1]);
  452. read(fds[0], &c, 1);
  453. if (opt_verbose)
  454. cerr << "backgrounding\n";
  455. exit(0);
  456. }
  457. close(fds[0]);
  458. opt_upbg_fd = fds[1];
  459. client(argc - optind, argv + optind);
  460. }
  461. else
  462. client(argc - optind, argv + optind);
  463. return 0;
  464. }