chiark - git - matthewv - gooswapper/blob - gooswapper.py

   1 #!/usr/bin/env python3
   2
   3 import sys
   4 import getpass
   5 import os
   6 import pickle
   7 import collections
   8 import argparse
   9 import time
  10 import logging
  11 logger = logging.getLogger('gooswapper')
  12 logger.setLevel(logging.INFO)
  13 consolelog = logging.StreamHandler()
  14 consolelog.setLevel(logging.INFO)
  15 logformatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
  16 consolelog.setFormatter(logformatter)
  17 logger.addHandler(consolelog)
  18 #We can't use this, because that way all the libraries' logs spam us
  19 #logging.basicConfig(level=logging.INFO)
  20
  21 #Exchange-related library
  22 sys.path.append("/upstreams/exchangelib")
  23 import exchangelib
  24
  25 #Google calendar-api libraries
  26 import httplib2
  27 import apiclient.discovery
  28 import oauth2client
  29 import oauth2client.file
  30 import oauth2client.client
  31 import googleapiclient.errors
  32
  33 #Not sure what the distribution approach here is...
  34 gcal_client_id = '805127902516-ptbbtgpq9o8pjr6r3k6hsm60j589o85u.apps.googleusercontent.com'
  35 gcal_client_secret = '8hpdxV3MauorryTDoZ1YK8JO'
  36
  37 #scope URL for r/w calendar access
  38 scope = 'https://www.googleapis.com/auth/calendar'
  39 #flow object, for doing OAuth2.0 stuff
  40 flow = oauth2client.client.OAuth2WebServerFlow(gcal_client_id,
  41                                                gcal_client_secret,
  42                                                scope)
  43
  44
  45 gcal_authpath=".gooswap_gcal_creds.dat"
  46
  47 cachepath=".gooswapcache"
  48
  49 exchange_credential = None
  50
  51 CachedExEvent=collections.namedtuple('CachedExEvent',
  52                                      ['changekey','gcal_link'])
  53
  54 class ex_gcal_link(exchangelib.ExtendedProperty):
  55     distinguished_property_set_id = 'PublicStrings'
  56     property_name = "google calendar event id"
  57     property_type = 'String'
  58
  59 try:
  60     exchangelib.CalendarItem.get_field_by_fieldname('gcal_link')
  61 except ValueError:
  62     exchangelib.CalendarItem.register('gcal_link',ex_gcal_link)
  63
  64 #useful if you want to replay an event
  65 def drop_from_ex_cache(itemid):
  66     with open(cachepath,"rb") as f:
  67         cache = pickle.load(f)
  68     cache.pop(itemid)
  69     with open(cachepath,"wb") as f:
  70         pickle.dump(cache,f)
  71
  72 def get_ex_event_by_itemid(calendar,itemid):
  73     return calendar.get(item_id=itemid)
  74
  75 def get_ex_event_by_id_and_changekey(acct,itemid,changekey):
  76     l=list(acct.fetch([(itemid,changekey)]))
  77     return list(acct.fetch([(itemid,changekey)]))[0]
  78
  79 def get_gcal_event_by_eventid(gcal_acct,eventId,gcal_id="primary"):
  80     return gcal_acct.events().get(calendarId=gcal_id,eventId=eventId).execute()
  81
  82 def get_gcal_recur_instance(gcal_acct,gcal_master,start,gcal_id="primary"):
  83     if gcal_master is None:
  84         logger.warning("Cannot get recurrences from event with null gcal id")
  85         return None
  86     ans = gcal_acct.events().instances(calendarId=gcal_id,
  87                                        eventId=gcal_master,
  88                                        originalStart=start.isoformat(),
  89                                        showDeleted=True).execute()
  90     if len(ans['items']) != 1:
  91         logger.error("Searching for recurrance instance returned %d events" % \
  92                      len(ans['items']))
  93         return None
  94     return ans['items'][0]
  95
  96 def get_ex_cred(username="SANGER\mv3",password=None):
  97     if password is None:
  98         password = getpass.getpass(prompt="Password for user %s: " % username)
  99     return exchangelib.ServiceAccount(username,password)
 100
 101 def ex_login(username,emailaddr,ad_cache_path=None):
 102     global exchange_credential
 103     autodiscover = True
 104     if exchange_credential is None:
 105         exchange_credential = get_ex_cred(username)
 106     if ad_cache_path is not None:
 107         try:
 108             with open(ad_cache_path,"rb") as f:
 109                 url,auth_type = pickle.load(f)
 110                 autodiscover = False
 111         except FileNotFoundError:
 112             pass
 113
 114     if autodiscover:
 115         ex_ac = exchangelib.Account(emailaddr,
 116                                     credentials = exchange_credential,
 117                                     autodiscover = autodiscover)
 118         if ad_cache_path is not None:
 119             cache=(ex_ac.protocol.service_endpoint,
 120                    ex_ac.protocol.auth_type)
 121             with open(ad_cache_path,"wb") as f:
 122                 pickle.dump(cache,f)
 123     else:
 124         ex_conf = exchangelib.Configuration(service_endpoint=url,
 125                                             credentials=exchange_credential,
 126                                             auth_type=auth_type)
 127         ex_ac = exchangelib.Account(emailaddr,
 128                                     config=ex_conf,
 129                                     autodiscover=False,
 130                                     access_type=exchangelib.DELEGATE)
 131
 132     return ex_ac
 133
 134 def get_ex_events(calendar):
 135     ans={}
 136     for event in calendar.all().only('changekey','item_id','gcal_link'):
 137         if event.item_id in ans:
 138             logger.warning("Event item_id %s was duplicated!" % event.item_id)
 139         ans[event.item_id] = CachedExEvent(event.changekey,event.gcal_link)
 140     logger.info("%d events found" % len(ans))
 141     return ans
 142
 143 def ex_event_changes(old,new):
 144     olds = set(old.keys())
 145     news = set(new.keys())
 146     added = list(news-olds)
 147     deleted = list(olds-news)
 148     changed = []
 149     #intersection - i.e. common to both sets
 150     for event in olds & news:
 151         if old[event].changekey != new[event].changekey:
 152             changed.append(event)
 153     logger.info("%d events updated, %d added, %d deleted" % (len(changed),
 154                                                               len(added),
 155                                                               len(deleted)))
 156     return added, deleted, changed
 157
 158 #exchangelib gives us days in recurrence patterns as integers,
 159 #RFC5545 wants SU,MO,TU,WE,TH,FR,SA
 160 #it has a utility function to convert to Monday, Tuesday, ...
 161 def rr_daystr_from_int(i):
 162     return exchangelib.recurrence._weekday_to_str(i).upper()[:2]
 163
 164 #for monthly patterns, we want the week (or -1 for last) combined with each
 165 #day specified
 166 def rr_daystr_monthly(p):
 167     if p.week_number == 5:
 168         wn = "-1"
 169     else:
 170         wn = str(p.week_number)
 171     return ",".join([wn + rr_daystr_from_int(x) for x in p.weekdays])
 172
 173 def rrule_from_ex(event,gcal_tz):
 174     if event.type != "RecurringMaster":
 175         logger.error("Cannot make recurrence from not-recurring event")
 176         return None
 177     if event.recurrence is None:
 178         logger.error("Empty recurrence structure")
 179         return None
 180     if isinstance(event.recurrence.pattern,
 181                   exchangelib.recurrence.DailyPattern):
 182         rr = "RRULE:FREQ=DAILY;INTERVAL=%d" % event.recurrence.pattern.interval
 183     elif isinstance(event.recurrence.pattern,
 184                     exchangelib.recurrence.WeeklyPattern):
 185         rr = "RRULE:FREQ=WEEKLY;INTERVAL=%d;BYDAY=%s;WKST=%s" % \
 186                           (event.recurrence.pattern.interval,
 187                            ",".join([rr_daystr_from_int(x) for x in event.recurrence.pattern.weekdays]),
 188                            rr_daystr_from_int(event.recurrence.pattern.first_day_of_week) )
 189     elif isinstance(event.recurrence.pattern,
 190                     exchangelib.recurrence.RelativeMonthlyPattern):
 191         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYDAY=%s" % \
 192                      (event.recurrence.pattern.interval,
 193                       rr_daystr_monthly(event.recurrence.pattern))
 194     elif isinstance(event.recurrence.pattern,
 195                     exchangelib.recurrence.AbsoluteMonthlyPattern):
 196         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYMONTHDAY=%d" % \
 197                           (event.recurrence.pattern.interval,
 198                            event.recurrence.pattern.day_of_month)
 199     elif isinstance(event.recurrence.pattern,
 200                     exchangelib.recurrence.AbsoluteYearlyPattern):
 201         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYMONTHDAY=%d" % \
 202                           (event.recurrence.pattern.month,
 203                            event.recurrence.pattern.day_of_month)
 204     elif isinstance(event.recurrence.pattern,
 205                     exchangelib.recurrence.RelativeYearlyPattern):
 206         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYDAY=%s" % \
 207                           (event.recurrence.pattern.month,
 208                            rr_daystr_monthly(event.recurrence.pattern))
 209     else:
 210         logger.error("Recurrence %s not supported" % event.recurrence)
 211         return None
 212     if isinstance(event.recurrence.boundary,
 213                   exchangelib.recurrence.EndDatePattern):
 214         rr += ";UNTIL={0:%Y}{0:%m}{0:%d}".format(event.recurrence.boundary.end)
 215     elif isinstance(event.recurrence.boundary,
 216                     exchangelib.recurrence.NoEndPattern):
 217         pass #no end date to set
 218     else:
 219         logger.error("Recurrence %s not supported" % event.recurrence)
 220         return None
 221     return [rr]
 222
 223 def modify_recurring(ex_acct,gcal_acct,gcal_tz,
 224                      events,master,gcal_id="primary"):
 225     if master.modified_occurrences is not None:
 226         for mod in master.modified_occurrences:
 227             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 228                                                mod.original_start,gcal_id)
 229             if instance is None: #give up after first failure
 230                 return
 231             mod_event = get_ex_event_by_itemid(ex_acct.calendar,mod.item_id)
 232             gevent = build_gcal_event_from_ex(mod_event,gcal_tz)
 233             gevent = gcal_acct.events().update(calendarId=gcal_id,
 234                                                eventId=instance.get('id'),
 235                                                body=gevent,
 236                                                sendUpdates="none").execute()
 237             mod_event.gcal_link = gevent.get("id")
 238             mod_event.save(update_fields=["gcal_link"])
 239     if master.deleted_occurrences is not None:
 240         for d in master.deleted_occurrences:
 241             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 242                                                d.start,gcal_id)
 243             if instance is None: #give up after any failure
 244                 return
 245             if instance["status"] != "cancelled":
 246                 instance["status"]="cancelled"
 247                 gcal_acct.events().update(calendarId=gcal_id,
 248                                           eventId=instance.get('id'),
 249                                           body=instance,
 250                                           sendUpdates="none").execute()
 251
 252 def build_gcal_event_from_ex(event,gcal_tz):
 253     gevent={}
 254     gevent["summary"]=event.subject
 255     if event.is_all_day:
 256         gevent["end"]={"date": str(event.end.astimezone(gcal_tz).date())}
 257         gevent["start"]={"date": str(event.start.astimezone(gcal_tz).date())}
 258     else:
 259         gevent["end"]={"dateTime": event.end.astimezone(gcal_tz).isoformat(),
 260                        "timeZone": str(gcal_tz)}
 261         gevent["start"]={"dateTime": event.start.astimezone(gcal_tz).isoformat(),
 262                          "timeZone": str(gcal_tz)}
 263     if event.text_body is not None and event.text_body.strip() != '':
 264         gevent["description"] = event.text_body
 265     if event.location is not None:
 266         gevent["location"] = event.location
 267     gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 268     return gevent
 269
 270 def add_ex_to_gcal(ex_acct,
 271                    gcal_acct,gcal_tz,events,
 272                    added,
 273                    gcal_id="primary"):
 274     for ev_id in added:
 275         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 276         gevent = build_gcal_event_from_ex(event,gcal_tz)
 277         if event.type=="RecurringMaster":
 278             rr = rrule_from_ex(event,gcal_tz)
 279             if rr is not None:
 280                 gevent["recurrence"] = rr
 281                 print(gevent)
 282             else:
 283                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 284                 continue #don't make the gcal event
 285         gevent = gcal_acct.events().insert(calendarId=gcal_id,
 286                                            body=gevent).execute()
 287         event.gcal_link = gevent.get("id")
 288         event.save(update_fields=["gcal_link"])
 289         if event.type=="RecurringMaster" and (event.deleted_occurrences or \
 290                                               event.modified_occurrences):
 291             modify_recurring(ex_acct,gcal_acct,gcal_tz,
 292                              events,event,gcal_id)
 293             #changekey is updated by the above
 294             event.refresh()
 295         events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 296
 297 def del_ex_to_gcal(ex_acct, gcal_acct, events, deleted, gcal_id="primary"):
 298     for ev_id in deleted:
 299         if events[ev_id].gcal_link is not None:
 300             gevent = get_gcal_event_by_eventid(gcal_acct,
 301                                                events[ev_id].gcal_link,
 302                                                gcal_id)
 303             if gevent["status"] != "cancelled":
 304                 gcal_acct.events().delete(calendarId=gcal_id,
 305                                           eventId=events[ev_id].gcal_link,
 306                                           sendUpdates="none").execute()
 307
 308 def update_ex_to_gcal(ex_acct,
 309                       gcal_acct,gcal_tz,
 310                       events,changed,
 311                       gcal_id="primary"):
 312     for ev_id in changed:
 313         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 314         if event.gcal_link is None:
 315             logger.warning("Cannot apply update where event has no gcal link")
 316             continue
 317         gevent = build_gcal_event_from_ex(event,gcal_tz)
 318         if event.type=="RecurringMaster":
 319             rr = rrule_from_ex(event,gcal_tz)
 320             if rr is not None:
 321                 gevent["recurrence"] = rr
 322                 if event.deleted_occurrences or \
 323                    event.modified_occurrences:
 324                     modify_recurring(ex_acct,gcal_acct,gcal_tz,
 325                                      events,event,gcal_id)
 326                     event.refresh() #changekey is updated by the above
 327                     events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 328             else:
 329                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 330                 continue #don't make the gcal event
 331         try: #may fail if we don't own the event
 332             gevent = gcal_acct.events().update(calendarId=gcal_id,
 333                                                eventId=event.gcal_link,
 334                                                body=gevent,
 335                                                sendUpdates="none").execute()
 336         except googleapiclient.errors.HttpError as err:
 337             if err.resp.status == 403:
 338                 pass
 339
 340 def match_ex_to_gcal(ex_acct,gcal_acct,gcal_tz,events,gcal_id="primary",ignore_link=True):
 341     recur = 0
 342     matched = 0
 343     skipped = 0
 344     toadd = []
 345     for ev_id in events:
 346         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 347         if event.gcal_link is not None and ignore_link is False:
 348             skipped += 1
 349             continue
 350         missing=True
 351         matches = gcal_acct.events().list(calendarId=gcal_id,
 352                                           timeMin=event.start.isoformat(),
 353                                           timeMax=event.end.isoformat()).execute()
 354         for ge in matches['items']:
 355             if ( ge.get("summary") is None and event.subject is None ) or \
 356                ( ge.get("summary") is not None and event.subject is not None \
 357                  and ge['summary'].strip()==event.subject.strip()):
 358                 logger.info("Matching '%s' starting at %s" % (event.subject,
 359                                                               event.start.isoformat()))
 360                 event.gcal_link = ge['id']
 361                 event.save(update_fields=["gcal_link"])
 362                 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 363                 gevent = {}
 364                 gevent["start"] = ge["start"]
 365                 gevent["end"] = ge["end"]
 366                 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 367                 try:
 368                     gcal_acct.events().update(calendarId=gcal_id,
 369                                               eventId=event.gcal_link,
 370                                               body=gevent,
 371                                               sendUpdates="none").execute()
 372                 #this may fail if we don't own the event
 373                 except googleapiclient.errors.HttpError as err:
 374                     if err.resp.status == 403:
 375                         pass
 376                 matched += 1
 377                 missing = False
 378                 break
 379         if missing == True:
 380             toadd.append(ev_id)
 381     logger.info("Matched %d events, skipped %d with existing link, and %d recurring ones" % (matched,skipped,recur))
 382     return toadd
 383
 384 def get_gcal_cred(args):
 385     #each such file can only store a single credential
 386     storage = oauth2client.file.Storage(gcal_authpath)
 387     gcal_credential = storage.get()
 388     #if no credential found, or they're invalid (e.g. expired),
 389     #then get a new one; pass --noauth_local_webserver on the command line
 390     #if you don't want it to spawn a browser
 391     if gcal_credential is None or gcal_credential.invalid:
 392         gcal_credential = oauth2client.tools.run_flow(flow,
 393                                                       storage,
 394                                                       args)
 395     return gcal_credential
 396
 397 def gcal_login(args):
 398     gcal_credential = get_gcal_cred(args)
 399     # Object to handle http requests; could add proxy details
 400     http = httplib2.Http()
 401     http = gcal_credential.authorize(http)
 402     return apiclient.discovery.build('calendar', 'v3', http=http)
 403
 404 def get_gcal_timezone(gcal_account,calendarid="primary"):
 405     gcal = gcal_account.calendars().get(calendarId=calendarid).execute()
 406     return exchangelib.EWSTimeZone.timezone(gcal['timeZone'])
 407
 408 def main():
 409     ap=argparse.ArgumentParser(description="Gooswapper calendar sync",
 410                                parents=[oauth2client.tools.argparser])
 411     ap.add_argument("exchuser",help="Exchange user e.g. 'SANGER\mv3'")
 412     ap.add_argument("exchemail",
 413                     help="Exchange calendar email e.g. ISGGroup@sanger.ac.uk")
 414     ap.add_argument("-g","--gcalid",help="google Calendar ID")
 415     ap.add_argument("-l","--loop",help="keep running indefinitely",
 416                     action="store_true")
 417     args = ap.parse_args()
 418     if args.gcalid is None:
 419         gcal_id = "primary"
 420     else:
 421         gcal_id = args.gcalid
 422
 423     #log in to the accounts
 424     ex_account = ex_login(args.exchuser,args.exchemail,
 425                           ".gooswapper_exch_conf.dat")
 426     gcal_account = gcal_login(args)
 427     gcal_tz = get_gcal_timezone(gcal_account,gcal_id)
 428
 429     #Main loop (broken at the end if login is false)
 430     while True:
 431         try:
 432             with open(cachepath,"rb") as f:
 433                 cache = pickle.load(f)
 434         except FileNotFoundError:
 435             cache = None
 436
 437         current = get_ex_events(ex_account.calendar)
 438
 439         if cache is not None:
 440             added,deleted,changed = ex_event_changes(cache,current)
 441             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 442                            added,gcal_id)
 443             #delete op needs the "cache" set, as that has the link ids in
 444             #for events that are now deleted
 445             del_ex_to_gcal(ex_account,gcal_account,cache,deleted,gcal_id)
 446             update_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 447                               changed,gcal_id)
 448         else:
 449             toadd = match_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 450                                      gcal_id)
 451             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 452                            toadd,gcal_id)
 453
 454         with open(cachepath,"wb") as f:
 455             pickle.dump(current,f)
 456
 457         #If not looping, break here (after 1 run)
 458         if args.loop==False:
 459             break
 460         #otherwise, wait 10 minutes, then go round again
 461         time.sleep(600)
 462
 463 if __name__ == "__main__":
 464     main()
 465
 466