chiark - git - matthewv - gooswapper/blob - gooswapper.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2018 Genome Research Limited
   4 #
   5 # Author: Matthew Vernon <mv3@sanger.ac.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published
   9 # by the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import getpass
  22 import os
  23 import pickle
  24 import collections
  25 import argparse
  26 import time
  27 import logging
  28 logger = logging.getLogger('gooswapper')
  29 logger.setLevel(logging.INFO)
  30 consolelog = logging.StreamHandler()
  31 consolelog.setLevel(logging.INFO)
  32 logformatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
  33 consolelog.setFormatter(logformatter)
  34 logger.addHandler(consolelog)
  35 #We can't use this, because that way all the libraries' logs spam us
  36 #logging.basicConfig(level=logging.INFO)
  37
  38 #Exchange-related library
  39 sys.path.append("/upstreams/exchangelib")
  40 import exchangelib
  41
  42 #Google calendar-api libraries
  43 import httplib2
  44 import apiclient.discovery
  45 import oauth2client
  46 import oauth2client.file
  47 import oauth2client.client
  48 import googleapiclient.errors
  49
  50 #Not sure what the distribution approach here is...
  51 gcal_client_id = '805127902516-ptbbtgpq9o8pjr6r3k6hsm60j589o85u.apps.googleusercontent.com'
  52 gcal_client_secret = '8hpdxV3MauorryTDoZ1YK8JO'
  53
  54 #scope URL for r/w calendar access
  55 scope = 'https://www.googleapis.com/auth/calendar'
  56 #flow object, for doing OAuth2.0 stuff
  57 flow = oauth2client.client.OAuth2WebServerFlow(gcal_client_id,
  58                                                gcal_client_secret,
  59                                                scope)
  60
  61
  62 gcal_authpath=".gooswap_gcal_creds.dat"
  63
  64 cachepath=".gooswapcache"
  65
  66 exchange_credential = None
  67
  68 CachedExEvent=collections.namedtuple('CachedExEvent',
  69                                      ['changekey','gcal_link'])
  70
  71 class ex_gcal_link(exchangelib.ExtendedProperty):
  72     distinguished_property_set_id = 'PublicStrings'
  73     property_name = "google calendar event id"
  74     property_type = 'String'
  75
  76 try:
  77     exchangelib.CalendarItem.get_field_by_fieldname('gcal_link')
  78 except ValueError:
  79     exchangelib.CalendarItem.register('gcal_link',ex_gcal_link)
  80
  81 #useful if you want to replay an event
  82 def drop_from_ex_cache(itemid):
  83     with open(cachepath,"rb") as f:
  84         cache = pickle.load(f)
  85     cache.pop(itemid)
  86     with open(cachepath,"wb") as f:
  87         pickle.dump(cache,f)
  88
  89 def get_ex_event_by_itemid(calendar,itemid):
  90     return calendar.get(item_id=itemid)
  91
  92 def get_ex_event_by_id_and_changekey(acct,itemid,changekey):
  93     l=list(acct.fetch([(itemid,changekey)]))
  94     return list(acct.fetch([(itemid,changekey)]))[0]
  95
  96 def get_gcal_event_by_eventid(gcal_acct,eventId,gcal_id="primary"):
  97     return gcal_acct.events().get(calendarId=gcal_id,eventId=eventId).execute()
  98
  99 def get_gcal_recur_instance(gcal_acct,gcal_master,start,gcal_id="primary"):
 100     if gcal_master is None:
 101         logger.warning("Cannot get recurrences from event with null gcal id")
 102         return None
 103     ans = gcal_acct.events().instances(calendarId=gcal_id,
 104                                        eventId=gcal_master,
 105                                        originalStart=start.isoformat(),
 106                                        showDeleted=True).execute()
 107     if len(ans['items']) != 1:
 108         logger.error("Searching for recurrance instance returned %d events" % \
 109                      len(ans['items']))
 110         return None
 111     return ans['items'][0]
 112
 113 def get_ex_cred(username="SANGER\mv3",password=None):
 114     if password is None:
 115         password = getpass.getpass(prompt="Password for user %s: " % username)
 116     return exchangelib.ServiceAccount(username,password)
 117
 118 def ex_login(username,emailaddr,ad_cache_path=None):
 119     global exchange_credential
 120     autodiscover = True
 121     if exchange_credential is None:
 122         exchange_credential = get_ex_cred(username)
 123     if ad_cache_path is not None:
 124         try:
 125             with open(ad_cache_path,"rb") as f:
 126                 url,auth_type = pickle.load(f)
 127                 autodiscover = False
 128         except FileNotFoundError:
 129             pass
 130
 131     if autodiscover:
 132         ex_ac = exchangelib.Account(emailaddr,
 133                                     credentials = exchange_credential,
 134                                     autodiscover = autodiscover)
 135         if ad_cache_path is not None:
 136             cache=(ex_ac.protocol.service_endpoint,
 137                    ex_ac.protocol.auth_type)
 138             with open(ad_cache_path,"wb") as f:
 139                 pickle.dump(cache,f)
 140     else:
 141         ex_conf = exchangelib.Configuration(service_endpoint=url,
 142                                             credentials=exchange_credential,
 143                                             auth_type=auth_type)
 144         ex_ac = exchangelib.Account(emailaddr,
 145                                     config=ex_conf,
 146                                     autodiscover=False,
 147                                     access_type=exchangelib.DELEGATE)
 148
 149     return ex_ac
 150
 151 def get_ex_events(calendar):
 152     ans={}
 153     for event in calendar.all().only('changekey','item_id','gcal_link'):
 154         if event.item_id in ans:
 155             logger.warning("Event item_id %s was duplicated!" % event.item_id)
 156         ans[event.item_id] = CachedExEvent(event.changekey,event.gcal_link)
 157     logger.info("%d events found" % len(ans))
 158     return ans
 159
 160 def ex_event_changes(old,new):
 161     olds = set(old.keys())
 162     news = set(new.keys())
 163     added = list(news-olds)
 164     deleted = list(olds-news)
 165     changed = []
 166     #intersection - i.e. common to both sets
 167     for event in olds & news:
 168         if old[event].changekey != new[event].changekey:
 169             changed.append(event)
 170     logger.info("%d events updated, %d added, %d deleted" % (len(changed),
 171                                                               len(added),
 172                                                               len(deleted)))
 173     return added, deleted, changed
 174
 175 #exchangelib gives us days in recurrence patterns as integers,
 176 #RFC5545 wants SU,MO,TU,WE,TH,FR,SA
 177 #it has a utility function to convert to Monday, Tuesday, ...
 178 def rr_daystr_from_int(i):
 179     return exchangelib.recurrence._weekday_to_str(i).upper()[:2]
 180
 181 #for monthly patterns, we want the week (or -1 for last) combined with each
 182 #day specified
 183 def rr_daystr_monthly(p):
 184     if p.week_number == 5:
 185         wn = "-1"
 186     else:
 187         wn = str(p.week_number)
 188     return ",".join([wn + rr_daystr_from_int(x) for x in p.weekdays])
 189
 190 def rrule_from_ex(event,gcal_tz):
 191     if event.type != "RecurringMaster":
 192         logger.error("Cannot make recurrence from not-recurring event")
 193         return None
 194     if event.recurrence is None:
 195         logger.error("Empty recurrence structure")
 196         return None
 197     if isinstance(event.recurrence.pattern,
 198                   exchangelib.recurrence.DailyPattern):
 199         rr = "RRULE:FREQ=DAILY;INTERVAL=%d" % event.recurrence.pattern.interval
 200     elif isinstance(event.recurrence.pattern,
 201                     exchangelib.recurrence.WeeklyPattern):
 202         rr = "RRULE:FREQ=WEEKLY;INTERVAL=%d;BYDAY=%s;WKST=%s" % \
 203                           (event.recurrence.pattern.interval,
 204                            ",".join([rr_daystr_from_int(x) for x in event.recurrence.pattern.weekdays]),
 205                            rr_daystr_from_int(event.recurrence.pattern.first_day_of_week) )
 206     elif isinstance(event.recurrence.pattern,
 207                     exchangelib.recurrence.RelativeMonthlyPattern):
 208         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYDAY=%s" % \
 209                      (event.recurrence.pattern.interval,
 210                       rr_daystr_monthly(event.recurrence.pattern))
 211     elif isinstance(event.recurrence.pattern,
 212                     exchangelib.recurrence.AbsoluteMonthlyPattern):
 213         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYMONTHDAY=%d" % \
 214                           (event.recurrence.pattern.interval,
 215                            event.recurrence.pattern.day_of_month)
 216     elif isinstance(event.recurrence.pattern,
 217                     exchangelib.recurrence.AbsoluteYearlyPattern):
 218         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYMONTHDAY=%d" % \
 219                           (event.recurrence.pattern.month,
 220                            event.recurrence.pattern.day_of_month)
 221     elif isinstance(event.recurrence.pattern,
 222                     exchangelib.recurrence.RelativeYearlyPattern):
 223         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYDAY=%s" % \
 224                           (event.recurrence.pattern.month,
 225                            rr_daystr_monthly(event.recurrence.pattern))
 226     else:
 227         logger.error("Recurrence %s not supported" % event.recurrence)
 228         return None
 229     if isinstance(event.recurrence.boundary,
 230                   exchangelib.recurrence.EndDatePattern):
 231         rr += ";UNTIL={0:%Y}{0:%m}{0:%d}".format(event.recurrence.boundary.end)
 232     elif isinstance(event.recurrence.boundary,
 233                     exchangelib.recurrence.NoEndPattern):
 234         pass #no end date to set
 235     else:
 236         logger.error("Recurrence %s not supported" % event.recurrence)
 237         return None
 238     return [rr]
 239
 240 def modify_recurring(ex_acct,gcal_acct,gcal_tz,
 241                      events,master,gcal_id="primary"):
 242     if master.modified_occurrences is not None:
 243         for mod in master.modified_occurrences:
 244             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 245                                                mod.original_start,gcal_id)
 246             if instance is None: #give up after first failure
 247                 return
 248             mod_event = get_ex_event_by_itemid(ex_acct.calendar,mod.item_id)
 249             gevent = build_gcal_event_from_ex(mod_event,gcal_tz)
 250             gevent = gcal_acct.events().update(calendarId=gcal_id,
 251                                                eventId=instance.get('id'),
 252                                                body=gevent,
 253                                                sendUpdates="none").execute()
 254             mod_event.gcal_link = gevent.get("id")
 255             mod_event.save(update_fields=["gcal_link"])
 256     if master.deleted_occurrences is not None:
 257         for d in master.deleted_occurrences:
 258             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 259                                                d.start,gcal_id)
 260             if instance is None: #give up after any failure
 261                 return
 262             if instance["status"] != "cancelled":
 263                 instance["status"]="cancelled"
 264                 gcal_acct.events().update(calendarId=gcal_id,
 265                                           eventId=instance.get('id'),
 266                                           body=instance,
 267                                           sendUpdates="none").execute()
 268
 269 def build_gcal_event_from_ex(event,gcal_tz):
 270     gevent={}
 271     gevent["summary"]=event.subject
 272     if event.is_all_day:
 273         gevent["end"]={"date": str(event.end.astimezone(gcal_tz).date())}
 274         gevent["start"]={"date": str(event.start.astimezone(gcal_tz).date())}
 275     else:
 276         gevent["end"]={"dateTime": event.end.astimezone(gcal_tz).isoformat(),
 277                        "timeZone": str(gcal_tz)}
 278         gevent["start"]={"dateTime": event.start.astimezone(gcal_tz).isoformat(),
 279                          "timeZone": str(gcal_tz)}
 280     if event.text_body is not None and event.text_body.strip() != '':
 281         gevent["description"] = event.text_body
 282     if event.location is not None:
 283         gevent["location"] = event.location
 284     gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 285     return gevent
 286
 287 def add_ex_to_gcal(ex_acct,
 288                    gcal_acct,gcal_tz,events,
 289                    added,
 290                    gcal_id="primary"):
 291     for ev_id in added:
 292         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 293         gevent = build_gcal_event_from_ex(event,gcal_tz)
 294         if event.type=="RecurringMaster":
 295             rr = rrule_from_ex(event,gcal_tz)
 296             if rr is not None:
 297                 gevent["recurrence"] = rr
 298                 print(gevent)
 299             else:
 300                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 301                 continue #don't make the gcal event
 302         gevent = gcal_acct.events().insert(calendarId=gcal_id,
 303                                            body=gevent).execute()
 304         event.gcal_link = gevent.get("id")
 305         event.save(update_fields=["gcal_link"])
 306         if event.type=="RecurringMaster" and (event.deleted_occurrences or \
 307                                               event.modified_occurrences):
 308             modify_recurring(ex_acct,gcal_acct,gcal_tz,
 309                              events,event,gcal_id)
 310             #changekey is updated by the above
 311             event.refresh()
 312         events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 313
 314 def del_ex_to_gcal(ex_acct, gcal_acct, events, deleted, gcal_id="primary"):
 315     for ev_id in deleted:
 316         if events[ev_id].gcal_link is not None:
 317             gevent = get_gcal_event_by_eventid(gcal_acct,
 318                                                events[ev_id].gcal_link,
 319                                                gcal_id)
 320             if gevent["status"] != "cancelled":
 321                 gcal_acct.events().delete(calendarId=gcal_id,
 322                                           eventId=events[ev_id].gcal_link,
 323                                           sendUpdates="none").execute()
 324
 325 def update_ex_to_gcal(ex_acct,
 326                       gcal_acct,gcal_tz,
 327                       events,changed,
 328                       gcal_id="primary"):
 329     for ev_id in changed:
 330         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 331         if event.gcal_link is None:
 332             logger.warning("Cannot apply update where event has no gcal link")
 333             continue
 334         gevent = build_gcal_event_from_ex(event,gcal_tz)
 335         if event.type=="RecurringMaster":
 336             rr = rrule_from_ex(event,gcal_tz)
 337             if rr is not None:
 338                 gevent["recurrence"] = rr
 339                 if event.deleted_occurrences or \
 340                    event.modified_occurrences:
 341                     modify_recurring(ex_acct,gcal_acct,gcal_tz,
 342                                      events,event,gcal_id)
 343                     event.refresh() #changekey is updated by the above
 344                     events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 345             else:
 346                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 347                 continue #don't make the gcal event
 348         try: #may fail if we don't own the event
 349             gevent = gcal_acct.events().update(calendarId=gcal_id,
 350                                                eventId=event.gcal_link,
 351                                                body=gevent,
 352                                                sendUpdates="none").execute()
 353         except googleapiclient.errors.HttpError as err:
 354             if err.resp.status == 403:
 355                 pass
 356
 357 def match_ex_to_gcal(ex_acct,gcal_acct,gcal_tz,events,gcal_id="primary",ignore_link=True):
 358     recur = 0
 359     matched = 0
 360     skipped = 0
 361     toadd = []
 362     for ev_id in events:
 363         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 364         if event.gcal_link is not None and ignore_link is False:
 365             skipped += 1
 366             continue
 367         missing=True
 368         matches = gcal_acct.events().list(calendarId=gcal_id,
 369                                           timeMin=event.start.isoformat(),
 370                                           timeMax=event.end.isoformat()).execute()
 371         for ge in matches['items']:
 372             if ( ge.get("summary") is None and event.subject is None ) or \
 373                ( ge.get("summary") is not None and event.subject is not None \
 374                  and ge['summary'].strip()==event.subject.strip()):
 375                 logger.info("Matching '%s' starting at %s" % (event.subject,
 376                                                               event.start.isoformat()))
 377                 event.gcal_link = ge['id']
 378                 event.save(update_fields=["gcal_link"])
 379                 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 380                 gevent = {}
 381                 gevent["start"] = ge["start"]
 382                 gevent["end"] = ge["end"]
 383                 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 384                 try:
 385                     gcal_acct.events().update(calendarId=gcal_id,
 386                                               eventId=event.gcal_link,
 387                                               body=gevent,
 388                                               sendUpdates="none").execute()
 389                 #this may fail if we don't own the event
 390                 except googleapiclient.errors.HttpError as err:
 391                     if err.resp.status == 403:
 392                         pass
 393                 matched += 1
 394                 missing = False
 395                 break
 396         if missing == True:
 397             toadd.append(ev_id)
 398     logger.info("Matched %d events, skipped %d with existing link, and %d recurring ones" % (matched,skipped,recur))
 399     return toadd
 400
 401 def get_gcal_cred(args):
 402     #each such file can only store a single credential
 403     storage = oauth2client.file.Storage(gcal_authpath)
 404     gcal_credential = storage.get()
 405     #if no credential found, or they're invalid (e.g. expired),
 406     #then get a new one; pass --noauth_local_webserver on the command line
 407     #if you don't want it to spawn a browser
 408     if gcal_credential is None or gcal_credential.invalid:
 409         gcal_credential = oauth2client.tools.run_flow(flow,
 410                                                       storage,
 411                                                       args)
 412     return gcal_credential
 413
 414 def gcal_login(args):
 415     gcal_credential = get_gcal_cred(args)
 416     # Object to handle http requests; could add proxy details
 417     http = httplib2.Http()
 418     http = gcal_credential.authorize(http)
 419     return apiclient.discovery.build('calendar', 'v3', http=http)
 420
 421 def get_gcal_timezone(gcal_account,calendarid="primary"):
 422     gcal = gcal_account.calendars().get(calendarId=calendarid).execute()
 423     return exchangelib.EWSTimeZone.timezone(gcal['timeZone'])
 424
 425 def main():
 426     ap=argparse.ArgumentParser(description="Gooswapper calendar sync",
 427                                parents=[oauth2client.tools.argparser])
 428     ap.add_argument("exchuser",help="Exchange user e.g. 'SANGER\mv3'")
 429     ap.add_argument("exchemail",
 430                     help="Exchange calendar email e.g. ISGGroup@sanger.ac.uk")
 431     ap.add_argument("-g","--gcalid",help="google Calendar ID")
 432     ap.add_argument("-l","--loop",help="keep running indefinitely",
 433                     action="store_true")
 434     args = ap.parse_args()
 435     if args.gcalid is None:
 436         gcal_id = "primary"
 437     else:
 438         gcal_id = args.gcalid
 439
 440     #log in to the accounts
 441     ex_account = ex_login(args.exchuser,args.exchemail,
 442                           ".gooswapper_exch_conf.dat")
 443     gcal_account = gcal_login(args)
 444     gcal_tz = get_gcal_timezone(gcal_account,gcal_id)
 445
 446     #Main loop (broken at the end if login is false)
 447     while True:
 448         try:
 449             with open(cachepath,"rb") as f:
 450                 cache = pickle.load(f)
 451         except FileNotFoundError:
 452             cache = None
 453
 454         current = get_ex_events(ex_account.calendar)
 455
 456         if cache is not None:
 457             added,deleted,changed = ex_event_changes(cache,current)
 458             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 459                            added,gcal_id)
 460             #delete op needs the "cache" set, as that has the link ids in
 461             #for events that are now deleted
 462             del_ex_to_gcal(ex_account,gcal_account,cache,deleted,gcal_id)
 463             update_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 464                               changed,gcal_id)
 465         else:
 466             toadd = match_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 467                                      gcal_id)
 468             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 469                            toadd,gcal_id)
 470
 471         with open(cachepath,"wb") as f:
 472             pickle.dump(current,f)
 473
 474         #If not looping, break here (after 1 run)
 475         if args.loop==False:
 476             break
 477         #otherwise, wait 10 minutes, then go round again
 478         time.sleep(600)
 479
 480 if __name__ == "__main__":
 481     main()
 482
 483