chiark - git - matthewv - gooswapper/blob - gooswapper.py

   1 #!/usr/bin/env python3
   2
   3 import sys
   4 import getpass
   5 import os
   6 import pickle
   7 import collections
   8 import argparse
   9 import logging
  10 logger = logging.getLogger('gooswapper')
  11 logger.setLevel(logging.INFO)
  12 consolelog = logging.StreamHandler()
  13 consolelog.setLevel(logging.INFO)
  14 logformatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
  15 consolelog.setFormatter(logformatter)
  16 logger.addHandler(consolelog)
  17 #We can't use this, because that way all the libraries' logs spam us
  18 #logging.basicConfig(level=logging.INFO)
  19
  20 #Exchange-related library
  21 sys.path.append("/upstreams/exchangelib")
  22 import exchangelib
  23
  24 #Google calendar-api libraries
  25 import httplib2
  26 import apiclient.discovery
  27 import oauth2client
  28 import oauth2client.file
  29 import oauth2client.client
  30 import googleapiclient.errors
  31
  32 #Not sure what the distribution approach here is...
  33 gcal_client_id = '805127902516-ptbbtgpq9o8pjr6r3k6hsm60j589o85u.apps.googleusercontent.com'
  34 gcal_client_secret = '8hpdxV3MauorryTDoZ1YK8JO'
  35
  36 #scope URL for r/w calendar access
  37 scope = 'https://www.googleapis.com/auth/calendar'
  38 #flow object, for doing OAuth2.0 stuff
  39 flow = oauth2client.client.OAuth2WebServerFlow(gcal_client_id,
  40                                                gcal_client_secret,
  41                                                scope)
  42
  43
  44 gcal_authpath=".gooswap_gcal_creds.dat"
  45
  46 cachepath=".gooswapcache"
  47
  48 exchange_credential = None
  49
  50 CachedExEvent=collections.namedtuple('CachedExEvent',
  51                                      ['changekey','gcal_link'])
  52
  53 class ex_gcal_link(exchangelib.ExtendedProperty):
  54     distinguished_property_set_id = 'PublicStrings'
  55     property_name = "google calendar event id"
  56     property_type = 'String'
  57
  58 try:
  59     exchangelib.CalendarItem.get_field_by_fieldname('gcal_link')
  60 except ValueError:
  61     exchangelib.CalendarItem.register('gcal_link',ex_gcal_link)
  62
  63 #useful if you want to replay an event
  64 def drop_from_ex_cache(itemid):
  65     with open(cachepath,"rb") as f:
  66         cache = pickle.load(f)
  67     cache.pop(itemid)
  68     with open(cachepath,"wb") as f:
  69         pickle.dump(cache,f)
  70
  71 def get_ex_event_by_itemid(calendar,itemid):
  72     return calendar.get(item_id=itemid)
  73
  74 def get_ex_event_by_id_and_changekey(acct,itemid,changekey):
  75     l=list(acct.fetch([(itemid,changekey)]))
  76     return list(acct.fetch([(itemid,changekey)]))[0]
  77
  78 def get_gcal_event_by_eventid(gcal_acct,eventId,gcal_id="primary"):
  79     return gcal_acct.events().get(calendarId=gcal_id,eventId=eventId).execute()
  80
  81 def get_gcal_recur_instance(gcal_acct,gcal_master,start,gcal_id="primary"):
  82     ans = gcal_acct.events().instances(calendarId=gcal_id,
  83                                        eventId=gcal_master,
  84                                        originalStart=start.isoformat(),
  85                                        showDeleted=True).execute()
  86     if len(ans['items']) != 1:
  87         logger.error("Searching for recurrance instance returned %d events" % \
  88                      len(ans['items']))
  89         return None
  90     return ans['items'][0]
  91
  92 def get_ex_cred(username="SANGER\mv3",password=None):
  93     if password is None:
  94         password = getpass.getpass(prompt="Password for user %s: " % username)
  95     return exchangelib.ServiceAccount(username,password)
  96
  97 def ex_login(username,emailaddr,ad_cache_path=None):
  98     global exchange_credential
  99     autodiscover = True
 100     if exchange_credential is None:
 101         exchange_credential = get_ex_cred(username)
 102     if ad_cache_path is not None:
 103         try:
 104             with open(ad_cache_path,"rb") as f:
 105                 url,auth_type = pickle.load(f)
 106                 autodiscover = False
 107         except FileNotFoundError:
 108             pass
 109
 110     if autodiscover:
 111         ex_ac = exchangelib.Account(emailaddr,
 112                                     credentials = exchange_credential,
 113                                     autodiscover = autodiscover)
 114         if ad_cache_path is not None:
 115             cache=(ex_ac.protocol.service_endpoint,
 116                    ex_ac.protocol.auth_type)
 117             with open(ad_cache_path,"wb") as f:
 118                 pickle.dump(cache,f)
 119     else:
 120         ex_conf = exchangelib.Configuration(service_endpoint=url,
 121                                             credentials=exchange_credential,
 122                                             auth_type=auth_type)
 123         ex_ac = exchangelib.Account(emailaddr,
 124                                     config=ex_conf,
 125                                     autodiscover=False,
 126                                     access_type=exchangelib.DELEGATE)
 127
 128     return ex_ac
 129
 130 def get_ex_events(calendar):
 131     ans={}
 132     for event in calendar.all().only('changekey','item_id','gcal_link'):
 133         if event.item_id in ans:
 134             logger.warning("Event item_id %s was duplicated!" % event.item_id)
 135         ans[event.item_id] = CachedExEvent(event.changekey,event.gcal_link)
 136     logger.info("%d events found" % len(ans))
 137     return ans
 138
 139 def ex_event_changes(old,new):
 140     olds = set(old.keys())
 141     news = set(new.keys())
 142     added = list(news-olds)
 143     deleted = list(olds-news)
 144     changed = []
 145     #intersection - i.e. common to both sets
 146     for event in olds & news:
 147         if old[event].changekey != new[event].changekey:
 148             changed.append(event)
 149     logger.info("%d events updated, %d added, %d deleted" % (len(changed),
 150                                                               len(added),
 151                                                               len(deleted)))
 152     return added, deleted, changed
 153
 154 #exchangelib gives us days in recurrence patterns as integers,
 155 #RFC5545 wants SU,MO,TU,WE,TH,FR,SA
 156 #it has a utility function to convert to Monday, Tuesday, ...
 157 def rr_daystr_from_int(i):
 158     return exchangelib.recurrence._weekday_to_str(i).upper()[:2]
 159
 160 #for monthly patterns, we want the week (or -1 for last) combined with each
 161 #day specified
 162 def rr_daystr_monthly(p):
 163     if p.week_number == 5:
 164         wn = "-1"
 165     else:
 166         wn = str(p.week_number)
 167     return ",".join([wn + rr_daystr_from_int(x) for x in p.weekdays])
 168
 169 def rrule_from_ex(event,gcal_tz):
 170     if event.type != "RecurringMaster":
 171         logger.error("Cannot make recurrence from not-recurring event")
 172         return None
 173     if event.recurrence is None:
 174         logger.error("Empty recurrence structure")
 175         return None
 176     if isinstance(event.recurrence.pattern,
 177                   exchangelib.recurrence.DailyPattern):
 178         rr = "RRULE:FREQ=DAILY;INTERVAL=%d" % event.recurrence.pattern.interval
 179     elif isinstance(event.recurrence.pattern,
 180                     exchangelib.recurrence.WeeklyPattern):
 181         rr = "RRULE:FREQ=WEEKLY;INTERVAL=%d;BYDAY=%s;WKST=%s" % \
 182                           (event.recurrence.pattern.interval,
 183                            ",".join([rr_daystr_from_int(x) for x in event.recurrence.pattern.weekdays]),
 184                            rr_daystr_from_int(event.recurrence.pattern.first_day_of_week) )
 185     elif isinstance(event.recurrence.pattern,
 186                     exchangelib.recurrence.RelativeMonthlyPattern):
 187         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYDAY=%s" % \
 188                      (event.recurrence.pattern.interval,
 189                       rr_daystr_monthly(event.recurrence.pattern))
 190     elif isinstance(event.recurrence.pattern,
 191                     exchangelib.recurrence.AbsoluteMonthlyPattern):
 192         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYMONTHDAY=%d" % \
 193                           (event.recurrence.pattern.interval,
 194                            event.recurrence.pattern.day_of_month)
 195     elif isinstance(event.recurrence.pattern,
 196                     exchangelib.recurrence.AbsoluteYearlyPattern):
 197         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYMONTHDAY=%d" % \
 198                           (event.recurrence.pattern.month,
 199                            event.recurrence.pattern.day_of_month)
 200     elif isinstance(event.recurrence.pattern,
 201                     exchangelib.recurrence.RelativeYearlyPattern):
 202         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYDAY=%s" % \
 203                           (event.recurrence.pattern.month,
 204                            rr_daystr_monthly(event.recurrence.pattern))
 205     else:
 206         logger.error("Recurrence %s not supported" % event.recurrence)
 207         return None
 208     if isinstance(event.recurrence.boundary,
 209                   exchangelib.recurrence.EndDatePattern):
 210         rr += ";UNTIL={0:%Y}{0:%m}{0:%d}".format(event.recurrence.boundary.end)
 211     elif isinstance(event.recurrence.boundary,
 212                     exchangelib.recurrence.NoEndPattern):
 213         pass #no end date to set
 214     else:
 215         logger.error("Recurrence %s not supported" % event.recurrence)
 216         return None
 217     return [rr]
 218
 219 def modify_recurring(ex_acct,gcal_acct,gcal_tz,
 220                      events,master,gcal_id="primary"):
 221     if master.modified_occurrences is not None:
 222         for mod in master.modified_occurrences:
 223             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 224                                                mod.original_start,gcal_id)
 225             if instance is None: #give up after first failure
 226                 return
 227             mod_event = get_ex_event_by_itemid(ex_acct.calendar,mod.item_id)
 228             gevent = build_gcal_event_from_ex(mod_event,gcal_tz)
 229             gevent = gcal_acct.events().update(calendarId=gcal_id,
 230                                                eventId=instance.get('id'),
 231                                                body=gevent,
 232                                                sendUpdates="none").execute()
 233             mod_event.gcal_link = gevent.get("id")
 234             mod_event.save(update_fields=["gcal_link"])
 235     if master.deleted_occurrences is not None:
 236         for d in master.deleted_occurrences:
 237             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 238                                                d.start,gcal_id)
 239             if instance is None: #give up after any failure
 240                 return
 241             if instance["status"] != "cancelled":
 242                 instance["status"]="cancelled"
 243                 gcal_acct.events().update(calendarId=gcal_id,
 244                                           eventId=instance.get('id'),
 245                                           body=instance,
 246                                           sendUpdates="none").execute()
 247
 248 def build_gcal_event_from_ex(event,gcal_tz):
 249     gevent={}
 250     gevent["summary"]=event.subject
 251     if event.is_all_day:
 252         gevent["end"]={"date": str(event.end.astimezone(gcal_tz).date())}
 253         gevent["start"]={"date": str(event.start.astimezone(gcal_tz).date())}
 254     else:
 255         gevent["end"]={"dateTime": event.end.astimezone(gcal_tz).isoformat(),
 256                        "timeZone": str(gcal_tz)}
 257         gevent["start"]={"dateTime": event.start.astimezone(gcal_tz).isoformat(),
 258                          "timeZone": str(gcal_tz)}
 259     if event.text_body is not None and event.text_body.strip() != '':
 260         gevent["description"] = event.text_body
 261     if event.location is not None:
 262         gevent["location"] = event.location
 263     gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 264     return gevent
 265
 266 def add_ex_to_gcal(ex_acct,
 267                    gcal_acct,gcal_tz,events,
 268                    added,
 269                    gcal_id="primary"):
 270     for ev_id in added:
 271         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 272         gevent = build_gcal_event_from_ex(event,gcal_tz)
 273         if event.type=="RecurringMaster":
 274             rr = rrule_from_ex(event,gcal_tz)
 275             if rr is not None:
 276                 gevent["recurrence"] = rr
 277                 print(gevent)
 278             else:
 279                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 280                 continue #don't make the gcal event
 281         gevent = gcal_acct.events().insert(calendarId=gcal_id,
 282                                            body=gevent).execute()
 283         event.gcal_link = gevent.get("id")
 284         event.save(update_fields=["gcal_link"])
 285         if event.type=="RecurringMaster" and (event.deleted_occurrences or \
 286                                               event.modified_occurrences):
 287             modify_recurring(ex_acct,gcal_acct,gcal_tz,
 288                              events,event,gcal_id)
 289             #changekey is updated by the above
 290             event.refresh()
 291         events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 292
 293 def del_ex_to_gcal(ex_acct, gcal_acct, events, deleted, gcal_id="primary"):
 294     for ev_id in deleted:
 295         if events[ev_id].gcal_link is not None:
 296             gevent = get_gcal_event_by_eventid(gcal_acct,
 297                                                events[ev_id].gcal_link,
 298                                                gcal_id)
 299             if gevent["status"] != "cancelled":
 300                 gcal_acct.events().delete(calendarId=gcal_id,
 301                                           eventId=events[ev_id].gcal_link,
 302                                           sendUpdates="none").execute()
 303
 304 def update_ex_to_gcal(ex_acct,
 305                       gcal_acct,gcal_tz,
 306                       events,changed,
 307                       gcal_id="primary"):
 308     for ev_id in changed:
 309         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 310         gevent = build_gcal_event_from_ex(event,gcal_tz)
 311         if event.type=="RecurringMaster":
 312             rr = rrule_from_ex(event,gcal_tz)
 313             if rr is not None:
 314                 gevent["recurrence"] = rr
 315                 if event.deleted_occurrences or \
 316                    event.modified_occurrences:
 317                     modify_recurring(ex_acct,gcal_acct,gcal_tz,
 318                                      events,event,gcal_id)
 319                     event.refresh() #changekey is updated by the above
 320                     events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 321             else:
 322                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 323                 continue #don't make the gcal event
 324         gevent = gcal_acct.events().update(calendarId=gcal_id,
 325                                                eventId=event.gcal_link,
 326                                                body=gevent,
 327                                                sendUpdates="none").execute()
 328
 329 def match_ex_to_gcal(ex_acct,gcal_acct,gcal_tz,events,gcal_id="primary",ignore_link=True):
 330     recur = 0
 331     matched = 0
 332     skipped = 0
 333     toadd = []
 334     for ev_id in events:
 335         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 336         if event.gcal_link is not None and ignore_link is False:
 337             skipped += 1
 338             continue
 339         missing=True
 340         matches = gcal_acct.events().list(calendarId=gcal_id,
 341                                           timeMin=event.start.isoformat(),
 342                                           timeMax=event.end.isoformat()).execute()
 343         for ge in matches['items']:
 344             if ( ge.get("summary") is None and event.subject is None ) or \
 345                ( ge.get("summary") is not None and event.subject is not None \
 346                  and ge['summary'].strip()==event.subject.strip()):
 347                 logger.info("Matching '%s' starting at %s" % (event.subject,
 348                                                               event.start.isoformat()))
 349                 event.gcal_link = ge['id']
 350                 event.save(update_fields=["gcal_link"])
 351                 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 352                 gevent = {}
 353                 gevent["start"] = ge["start"]
 354                 gevent["end"] = ge["end"]
 355                 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 356                 try:
 357                     gcal_acct.events().update(calendarId=gcal_id,
 358                                               eventId=event.gcal_link,
 359                                               body=gevent,
 360                                               sendUpdates="none").execute()
 361                 #this may fail if we don't own the event
 362                 except googleapiclient.errors.HttpError as err:
 363                     if err.resp.status == 403:
 364                         pass
 365                 matched += 1
 366                 missing = False
 367                 break
 368         if missing == True:
 369             toadd.append(ev_id)
 370     logger.info("Matched %d events, skipped %d with existing link, and %d recurring ones" % (matched,skipped,recur))
 371     return toadd
 372
 373 def get_gcal_cred(args):
 374     #each such file can only store a single credential
 375     storage = oauth2client.file.Storage(gcal_authpath)
 376     gcal_credential = storage.get()
 377     #if no credential found, or they're invalid (e.g. expired),
 378     #then get a new one; pass --noauth_local_webserver on the command line
 379     #if you don't want it to spawn a browser
 380     if gcal_credential is None or gcal_credential.invalid:
 381         gcal_credential = oauth2client.tools.run_flow(flow,
 382                                                       storage,
 383                                                       args)
 384     return gcal_credential
 385
 386 def gcal_login(args):
 387     gcal_credential = get_gcal_cred(args)
 388     # Object to handle http requests; could add proxy details
 389     http = httplib2.Http()
 390     http = gcal_credential.authorize(http)
 391     return apiclient.discovery.build('calendar', 'v3', http=http)
 392
 393 def get_gcal_timezone(gcal_account,calendarid="primary"):
 394     gcal = gcal_account.calendars().get(calendarId=calendarid).execute()
 395     return exchangelib.EWSTimeZone.timezone(gcal['timeZone'])
 396
 397 def main():
 398     ap=argparse.ArgumentParser(description="Gooswapper calendar sync",
 399                                parents=[oauth2client.tools.argparser])
 400     ap.add_argument("exchuser",help="Exchange user e.g. 'SANGER\mv3'")
 401     ap.add_argument("exchemail",
 402                     help="Exchange calendar email e.g. ISGGroup@sanger.ac.uk")
 403     ap.add_argument("-g","--gcalid",help="google Calendar ID")
 404     ap.add_argument("-l","--loop",help="keep running indefinitely",
 405                     action="store_true")
 406     args = ap.parse_args()
 407     if args.gcalid is None:
 408         gcal_id = "primary"
 409     else:
 410         gcal_id = args.gcalid
 411     try:
 412         with open(cachepath,"rb") as f:
 413             cache = pickle.load(f)
 414     except FileNotFoundError:
 415         cache = None
 416
 417     ex_account = ex_login(args.exchuser,args.exchemail,
 418                           ".gooswapper_exch_conf.dat")
 419     current = get_ex_events(ex_account.calendar)
 420
 421     gcal_account = gcal_login(args)
 422     gcal_tz = get_gcal_timezone(gcal_account,gcal_id)
 423
 424     if cache is not None:
 425         added,deleted,changed = ex_event_changes(cache,current)
 426         add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,added,gcal_id)
 427         #delete op needs the "cache" set, as that has the link ids in
 428         #for events that are now deleted
 429         del_ex_to_gcal(ex_account,gcal_account,cache,deleted,gcal_id)
 430         update_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 431                           changed,gcal_id)
 432     else:
 433         toadd = match_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 434                                  gcal_id)
 435         add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,toadd,gcal_id)
 436
 437     with open(cachepath,"wb") as f:
 438         pickle.dump(current,f)
 439
 440 if __name__ == "__main__":
 441     main()
 442
 443