3 # Copyright (C) 2018 Genome Research Limited
5 # Author: Matthew Vernon <mv3@sanger.ac.uk>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published
9 # by the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
28 logger = logging.getLogger('gooswapper')
29 logger.setLevel(logging.INFO)
30 consolelog = logging.StreamHandler()
31 consolelog.setLevel(logging.INFO)
32 logformatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
33 consolelog.setFormatter(logformatter)
34 logger.addHandler(consolelog)
35 #We can't use this, because that way all the libraries' logs spam us
36 #logging.basicConfig(level=logging.INFO)
38 #Exchange-related library
39 sys.path.append("/upstreams/exchangelib")
42 #Google calendar-api libraries
44 import apiclient.discovery
46 import oauth2client.file
47 import oauth2client.client
48 import googleapiclient.errors
50 #Not sure what the distribution approach here is...
51 gcal_client_id = '805127902516-ptbbtgpq9o8pjr6r3k6hsm60j589o85u.apps.googleusercontent.com'
52 gcal_client_secret = '8hpdxV3MauorryTDoZ1YK8JO'
54 #scope URL for r/w calendar access
55 scope = 'https://www.googleapis.com/auth/calendar'
56 #flow object, for doing OAuth2.0 stuff
57 flow = oauth2client.client.OAuth2WebServerFlow(gcal_client_id,
62 gcal_authpath=".gooswap_gcal_creds.dat"
64 cachepath=".gooswapcache"
66 exchange_credential = None
68 CachedExEvent=collections.namedtuple('CachedExEvent',
69 ['changekey','gcal_link'])
71 class ex_gcal_link(exchangelib.ExtendedProperty):
72 distinguished_property_set_id = 'PublicStrings'
73 property_name = "google calendar event id"
74 property_type = 'String'
77 exchangelib.CalendarItem.get_field_by_fieldname('gcal_link')
79 exchangelib.CalendarItem.register('gcal_link',ex_gcal_link)
81 #useful if you want to replay an event
82 def drop_from_ex_cache(itemid):
83 with open(cachepath,"rb") as f:
84 cache = pickle.load(f)
86 with open(cachepath,"wb") as f:
89 def get_ex_event_by_itemid(calendar,itemid):
90 return calendar.get(item_id=itemid)
92 def get_ex_event_by_id_and_changekey(acct,itemid,changekey):
93 l=list(acct.fetch([(itemid,changekey)]))
94 return list(acct.fetch([(itemid,changekey)]))[0]
96 def get_gcal_event_by_eventid(gcal_acct,eventId,gcal_id="primary"):
97 return gcal_acct.events().get(calendarId=gcal_id,eventId=eventId).execute()
99 def get_gcal_recur_instance(gcal_acct,gcal_master,start,gcal_id="primary"):
100 if gcal_master is None:
101 logger.warning("Cannot get recurrences from event with null gcal id")
103 ans = gcal_acct.events().instances(calendarId=gcal_id,
105 originalStart=start.isoformat(),
106 showDeleted=True).execute()
107 if len(ans['items']) != 1:
108 logger.error("Searching for recurrance instance returned %d events" % \
111 return ans['items'][0]
113 def get_ex_cred(username="SANGER\mv3",password=None):
115 password = getpass.getpass(prompt="Password for user %s: " % username)
116 return exchangelib.ServiceAccount(username,password)
118 def ex_login(username,emailaddr,ad_cache_path=None):
119 global exchange_credential
121 if exchange_credential is None:
122 exchange_credential = get_ex_cred(username)
123 if ad_cache_path is not None:
125 with open(ad_cache_path,"rb") as f:
126 url,auth_type = pickle.load(f)
128 except FileNotFoundError:
132 ex_ac = exchangelib.Account(emailaddr,
133 credentials = exchange_credential,
134 autodiscover = autodiscover)
135 if ad_cache_path is not None:
136 cache=(ex_ac.protocol.service_endpoint,
137 ex_ac.protocol.auth_type)
138 with open(ad_cache_path,"wb") as f:
141 ex_conf = exchangelib.Configuration(service_endpoint=url,
142 credentials=exchange_credential,
144 ex_ac = exchangelib.Account(emailaddr,
147 access_type=exchangelib.DELEGATE)
151 def get_ex_events(calendar):
153 for event in calendar.all().only('changekey','item_id','gcal_link'):
154 if event.item_id in ans:
155 logger.warning("Event item_id %s was duplicated!" % event.item_id)
156 ans[event.item_id] = CachedExEvent(event.changekey,event.gcal_link)
157 logger.info("%d events found" % len(ans))
160 def ex_event_changes(old,new):
161 olds = set(old.keys())
162 news = set(new.keys())
163 added = list(news-olds)
164 deleted = list(olds-news)
166 #intersection - i.e. common to both sets
167 for event in olds & news:
168 if old[event].changekey != new[event].changekey:
169 changed.append(event)
170 logger.info("%d events updated, %d added, %d deleted" % (len(changed),
173 return added, deleted, changed
175 #exchangelib gives us days in recurrence patterns as integers,
176 #RFC5545 wants SU,MO,TU,WE,TH,FR,SA
177 #it has a utility function to convert to Monday, Tuesday, ...
178 def rr_daystr_from_int(i):
179 return exchangelib.recurrence._weekday_to_str(i).upper()[:2]
181 #for monthly patterns, we want the week (or -1 for last) combined with each
183 def rr_daystr_monthly(p):
184 if p.week_number == 5:
187 wn = str(p.week_number)
188 return ",".join([wn + rr_daystr_from_int(x) for x in p.weekdays])
190 def rrule_from_ex(event,gcal_tz):
191 if event.type != "RecurringMaster":
192 logger.error("Cannot make recurrence from not-recurring event")
194 if event.recurrence is None:
195 logger.error("Empty recurrence structure")
197 if isinstance(event.recurrence.pattern,
198 exchangelib.recurrence.DailyPattern):
199 rr = "RRULE:FREQ=DAILY;INTERVAL=%d" % event.recurrence.pattern.interval
200 elif isinstance(event.recurrence.pattern,
201 exchangelib.recurrence.WeeklyPattern):
202 rr = "RRULE:FREQ=WEEKLY;INTERVAL=%d;BYDAY=%s;WKST=%s" % \
203 (event.recurrence.pattern.interval,
204 ",".join([rr_daystr_from_int(x) for x in event.recurrence.pattern.weekdays]),
205 rr_daystr_from_int(event.recurrence.pattern.first_day_of_week) )
206 elif isinstance(event.recurrence.pattern,
207 exchangelib.recurrence.RelativeMonthlyPattern):
208 rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYDAY=%s" % \
209 (event.recurrence.pattern.interval,
210 rr_daystr_monthly(event.recurrence.pattern))
211 elif isinstance(event.recurrence.pattern,
212 exchangelib.recurrence.AbsoluteMonthlyPattern):
213 rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYMONTHDAY=%d" % \
214 (event.recurrence.pattern.interval,
215 event.recurrence.pattern.day_of_month)
216 elif isinstance(event.recurrence.pattern,
217 exchangelib.recurrence.AbsoluteYearlyPattern):
218 rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYMONTHDAY=%d" % \
219 (event.recurrence.pattern.month,
220 event.recurrence.pattern.day_of_month)
221 elif isinstance(event.recurrence.pattern,
222 exchangelib.recurrence.RelativeYearlyPattern):
223 rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYDAY=%s" % \
224 (event.recurrence.pattern.month,
225 rr_daystr_monthly(event.recurrence.pattern))
227 logger.error("Recurrence %s not supported" % event.recurrence)
229 if isinstance(event.recurrence.boundary,
230 exchangelib.recurrence.EndDatePattern):
231 rr += ";UNTIL={0:%Y}{0:%m}{0:%d}".format(event.recurrence.boundary.end)
232 elif isinstance(event.recurrence.boundary,
233 exchangelib.recurrence.NoEndPattern):
234 pass #no end date to set
236 logger.error("Recurrence %s not supported" % event.recurrence)
240 def modify_recurring(ex_acct,gcal_acct,gcal_tz,
241 events,master,gcal_id="primary"):
242 if master.modified_occurrences is not None:
243 for mod in master.modified_occurrences:
244 instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
245 mod.original_start,gcal_id)
246 if instance is None: #give up after first failure
248 mod_event = get_ex_event_by_itemid(ex_acct.calendar,mod.item_id)
249 gevent = build_gcal_event_from_ex(mod_event,gcal_tz)
250 gevent = gcal_acct.events().update(calendarId=gcal_id,
251 eventId=instance.get('id'),
253 sendUpdates="none").execute()
254 mod_event.gcal_link = gevent.get("id")
255 mod_event.save(update_fields=["gcal_link"])
256 if master.deleted_occurrences is not None:
257 for d in master.deleted_occurrences:
258 instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
260 if instance is None: #give up after any failure
262 if instance["status"] != "cancelled":
263 instance["status"]="cancelled"
264 gcal_acct.events().update(calendarId=gcal_id,
265 eventId=instance.get('id'),
267 sendUpdates="none").execute()
269 def build_gcal_event_from_ex(event,gcal_tz):
271 gevent["summary"]=event.subject
273 gevent["end"]={"date": str(event.end.astimezone(gcal_tz).date())}
274 gevent["start"]={"date": str(event.start.astimezone(gcal_tz).date())}
276 gevent["end"]={"dateTime": event.end.astimezone(gcal_tz).isoformat(),
277 "timeZone": str(gcal_tz)}
278 gevent["start"]={"dateTime": event.start.astimezone(gcal_tz).isoformat(),
279 "timeZone": str(gcal_tz)}
280 if event.text_body is not None and event.text_body.strip() != '':
281 gevent["description"] = event.text_body
282 if event.location is not None:
283 gevent["location"] = event.location
284 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
287 def add_ex_to_gcal(ex_acct,
288 gcal_acct,gcal_tz,events,
292 event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
293 gevent = build_gcal_event_from_ex(event,gcal_tz)
294 if event.type=="RecurringMaster":
295 rr = rrule_from_ex(event,gcal_tz)
297 gevent["recurrence"] = rr
300 logger.warning("Unable to set recurrence for %s" % event.item_id)
301 continue #don't make the gcal event
302 gevent = gcal_acct.events().insert(calendarId=gcal_id,
303 body=gevent).execute()
304 event.gcal_link = gevent.get("id")
305 event.save(update_fields=["gcal_link"])
306 if event.type=="RecurringMaster" and (event.deleted_occurrences or \
307 event.modified_occurrences):
308 modify_recurring(ex_acct,gcal_acct,gcal_tz,
309 events,event,gcal_id)
310 #changekey is updated by the above
312 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
314 def del_ex_to_gcal(ex_acct, gcal_acct, events, deleted, gcal_id="primary"):
315 for ev_id in deleted:
316 if events[ev_id].gcal_link is not None:
317 gevent = get_gcal_event_by_eventid(gcal_acct,
318 events[ev_id].gcal_link,
320 if gevent["status"] != "cancelled":
321 gcal_acct.events().delete(calendarId=gcal_id,
322 eventId=events[ev_id].gcal_link,
323 sendUpdates="none").execute()
325 def update_ex_to_gcal(ex_acct,
329 for ev_id in changed:
330 event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
331 if event.gcal_link is None:
332 logger.warning("Cannot apply update where event has no gcal link")
334 gevent = build_gcal_event_from_ex(event,gcal_tz)
335 if event.type=="RecurringMaster":
336 rr = rrule_from_ex(event,gcal_tz)
338 gevent["recurrence"] = rr
339 if event.deleted_occurrences or \
340 event.modified_occurrences:
341 modify_recurring(ex_acct,gcal_acct,gcal_tz,
342 events,event,gcal_id)
343 event.refresh() #changekey is updated by the above
344 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
346 logger.warning("Unable to set recurrence for %s" % event.item_id)
347 continue #don't make the gcal event
348 try: #may fail if we don't own the event
349 gevent = gcal_acct.events().update(calendarId=gcal_id,
350 eventId=event.gcal_link,
352 sendUpdates="none").execute()
353 except googleapiclient.errors.HttpError as err:
354 if err.resp.status == 403:
357 def match_ex_to_gcal(ex_acct,gcal_acct,gcal_tz,events,gcal_id="primary",ignore_link=True):
363 event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
364 if event.gcal_link is not None and ignore_link is False:
368 matches = gcal_acct.events().list(calendarId=gcal_id,
369 timeMin=event.start.isoformat(),
370 timeMax=event.end.isoformat()).execute()
371 for ge in matches['items']:
372 if ( ge.get("summary") is None and event.subject is None ) or \
373 ( ge.get("summary") is not None and event.subject is not None \
374 and ge['summary'].strip()==event.subject.strip()):
375 logger.info("Matching '%s' starting at %s" % (event.subject,
376 event.start.isoformat()))
377 event.gcal_link = ge['id']
378 event.save(update_fields=["gcal_link"])
379 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
381 gevent["start"] = ge["start"]
382 gevent["end"] = ge["end"]
383 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
385 gcal_acct.events().update(calendarId=gcal_id,
386 eventId=event.gcal_link,
388 sendUpdates="none").execute()
389 #this may fail if we don't own the event
390 except googleapiclient.errors.HttpError as err:
391 if err.resp.status == 403:
398 logger.info("Matched %d events, skipped %d with existing link, and %d recurring ones" % (matched,skipped,recur))
401 def get_gcal_cred(args):
402 #each such file can only store a single credential
403 storage = oauth2client.file.Storage(gcal_authpath)
404 gcal_credential = storage.get()
405 #if no credential found, or they're invalid (e.g. expired),
406 #then get a new one; pass --noauth_local_webserver on the command line
407 #if you don't want it to spawn a browser
408 if gcal_credential is None or gcal_credential.invalid:
409 gcal_credential = oauth2client.tools.run_flow(flow,
412 return gcal_credential
414 def gcal_login(args):
415 gcal_credential = get_gcal_cred(args)
416 # Object to handle http requests; could add proxy details
417 http = httplib2.Http()
418 http = gcal_credential.authorize(http)
419 return apiclient.discovery.build('calendar', 'v3', http=http)
421 def get_gcal_timezone(gcal_account,calendarid="primary"):
422 gcal = gcal_account.calendars().get(calendarId=calendarid).execute()
423 return exchangelib.EWSTimeZone.timezone(gcal['timeZone'])
426 ap=argparse.ArgumentParser(description="Gooswapper calendar sync",
427 parents=[oauth2client.tools.argparser])
428 ap.add_argument("exchuser",help="Exchange user e.g. 'SANGER\mv3'")
429 ap.add_argument("exchemail",
430 help="Exchange calendar email e.g. ISGGroup@sanger.ac.uk")
431 ap.add_argument("-g","--gcalid",help="google Calendar ID")
432 ap.add_argument("-l","--loop",help="keep running indefinitely",
434 args = ap.parse_args()
435 if args.gcalid is None:
438 gcal_id = args.gcalid
440 #log in to the accounts
441 ex_account = ex_login(args.exchuser,args.exchemail,
442 ".gooswapper_exch_conf.dat")
443 gcal_account = gcal_login(args)
444 gcal_tz = get_gcal_timezone(gcal_account,gcal_id)
446 #Main loop (broken at the end if login is false)
449 with open(cachepath,"rb") as f:
450 cache = pickle.load(f)
451 except FileNotFoundError:
454 current = get_ex_events(ex_account.calendar)
456 if cache is not None:
457 added,deleted,changed = ex_event_changes(cache,current)
458 add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
460 #delete op needs the "cache" set, as that has the link ids in
461 #for events that are now deleted
462 del_ex_to_gcal(ex_account,gcal_account,cache,deleted,gcal_id)
463 update_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
466 toadd = match_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
468 add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
471 with open(cachepath,"wb") as f:
472 pickle.dump(current,f)
474 #If not looping, break here (after 1 run)
477 #otherwise, wait 10 minutes, then go round again
480 if __name__ == "__main__":