chiark - git - matthewv - gooswapper/blob - gooswapper.py

   1 #!/usr/bin/env python3
   2
   3 # Copyright (C) 2018 Genome Research Limited
   4 #
   5 # Author: Matthew Vernon <mv3@sanger.ac.uk>
   6 #
   7 # This program is free software: you can redistribute it and/or modify
   8 # it under the terms of the GNU Affero General Public License as published
   9 # by the Free Software Foundation, either version 3 of the License, or
  10 # (at your option) any later version.
  11 #
  12 # This program is distributed in the hope that it will be useful,
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 # GNU Affero General Public License for more details.
  16 #
  17 # You should have received a copy of the GNU Affero General Public License
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  19
  20 import sys
  21 import getpass
  22 import os
  23 import os.path
  24 import pickle
  25 import collections
  26 import argparse
  27 import time
  28 import logging
  29 logger = logging.getLogger('gooswapper')
  30 logger.setLevel(logging.INFO)
  31 consolelog = logging.StreamHandler()
  32 consolelog.setLevel(logging.INFO)
  33 logformatter = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
  34 consolelog.setFormatter(logformatter)
  35 logger.addHandler(consolelog)
  36 #We can't use this, because that way all the libraries' logs spam us
  37 #logging.basicConfig(level=logging.INFO)
  38
  39 #Exchange-related library
  40 sys.path.append("/upstreams/exchangelib")
  41 import exchangelib
  42
  43 #Google calendar-api libraries
  44 import httplib2
  45 import apiclient.discovery
  46 import oauth2client
  47 import oauth2client.file
  48 import oauth2client.client
  49 import googleapiclient.errors
  50
  51 #Not sure what the distribution approach here is...
  52 gcal_client_id = '805127902516-ptbbtgpq9o8pjr6r3k6hsm60j589o85u.apps.googleusercontent.com'
  53 gcal_client_secret = '8hpdxV3MauorryTDoZ1YK8JO'
  54
  55 #scope URL for r/w calendar access
  56 scope = 'https://www.googleapis.com/auth/calendar'
  57 #flow object, for doing OAuth2.0 stuff
  58 flow = oauth2client.client.OAuth2WebServerFlow(gcal_client_id,
  59                                                gcal_client_secret,
  60                                                scope)
  61
  62 gsdir = os.path.expanduser("~/.gooswapper")
  63 gcal_authpath = gsdir + "/.gooswap_gcal_creds.dat"
  64
  65 cachepath=None
  66
  67 exchange_credential = None
  68
  69 CachedExEvent=collections.namedtuple('CachedExEvent',
  70                                      ['changekey','gcal_link'])
  71
  72 class ex_gcal_link(exchangelib.ExtendedProperty):
  73     distinguished_property_set_id = 'PublicStrings'
  74     property_name = "google calendar event id"
  75     property_type = 'String'
  76
  77 try:
  78     exchangelib.CalendarItem.get_field_by_fieldname('gcal_link')
  79 except ValueError:
  80     exchangelib.CalendarItem.register('gcal_link',ex_gcal_link)
  81
  82 #useful if you want to replay an event
  83 def drop_from_ex_cache(itemid):
  84     with open(cachepath,"rb") as f:
  85         cache = pickle.load(f)
  86     cache.pop(itemid)
  87     with open(cachepath,"wb") as f:
  88         pickle.dump(cache,f)
  89
  90 def get_ex_event_by_itemid(calendar,itemid):
  91     return calendar.get(item_id=itemid)
  92
  93 def get_ex_event_by_id_and_changekey(acct,itemid,changekey):
  94     l=list(acct.fetch([(itemid,changekey)]))
  95     return list(acct.fetch([(itemid,changekey)]))[0]
  96
  97 def get_gcal_event_by_eventid(gcal_acct,eventId,gcal_id="primary"):
  98     return gcal_acct.events().get(calendarId=gcal_id,eventId=eventId).execute()
  99
 100 def get_gcal_recur_instance(gcal_acct,gcal_master,start,is_all_day,stz,
 101                             gcal_id="primary"):
 102     if gcal_master is None:
 103         logger.warning("Cannot get recurrences from event with null gcal id")
 104         return None
 105     if is_all_day:
 106         os = str(start.astimezone(stz).date())
 107     else:
 108         os = start.astimezone(stz).isoformat()
 109     ans = gcal_acct.events().instances(calendarId=gcal_id,
 110                                        eventId=gcal_master,
 111                                        originalStart=os,
 112                                        showDeleted=True).execute()
 113     if len(ans['items']) != 1:
 114         logger.error("Searching for recurrance instance returned %d events" % \
 115                      len(ans['items']))
 116         return None
 117     return ans['items'][0]
 118
 119 def get_ex_cred(username="SANGER\mv3",password=None):
 120     if password is None:
 121         password = getpass.getpass(prompt="Password for user %s: " % username)
 122     return exchangelib.ServiceAccount(username,password)
 123
 124 def ex_login(username,emailaddr,ad_cache_path=None):
 125     global exchange_credential
 126     autodiscover = True
 127     if exchange_credential is None:
 128         exchange_credential = get_ex_cred(username)
 129     if ad_cache_path is not None:
 130         try:
 131             with open(ad_cache_path,"rb") as f:
 132                 url,auth_type = pickle.load(f)
 133                 autodiscover = False
 134         except FileNotFoundError:
 135             pass
 136
 137     if autodiscover:
 138         ex_ac = exchangelib.Account(emailaddr,
 139                                     credentials = exchange_credential,
 140                                     autodiscover = autodiscover)
 141         if ad_cache_path is not None:
 142             cache=(ex_ac.protocol.service_endpoint,
 143                    ex_ac.protocol.auth_type)
 144             with open(ad_cache_path,"wb") as f:
 145                 pickle.dump(cache,f)
 146     else:
 147         ex_conf = exchangelib.Configuration(service_endpoint=url,
 148                                             credentials=exchange_credential,
 149                                             auth_type=auth_type)
 150         ex_ac = exchangelib.Account(emailaddr,
 151                                     config=ex_conf,
 152                                     autodiscover=False,
 153                                     access_type=exchangelib.DELEGATE)
 154
 155     return ex_ac
 156
 157 def get_ex_events(calendar):
 158     ans={}
 159     for event in calendar.all().only('changekey','item_id','gcal_link'):
 160         if event.item_id in ans:
 161             logger.warning("Event item_id %s was duplicated!" % event.item_id)
 162         ans[event.item_id] = CachedExEvent(event.changekey,event.gcal_link)
 163     logger.info("%d events found" % len(ans))
 164     return ans
 165
 166 def ex_event_changes(old,new):
 167     olds = set(old.keys())
 168     news = set(new.keys())
 169     added = list(news-olds)
 170     deleted = list(olds-news)
 171     changed = []
 172     #intersection - i.e. common to both sets
 173     for event in olds & news:
 174         if old[event].changekey != new[event].changekey:
 175             changed.append(event)
 176     logger.info("%d events updated, %d added, %d deleted" % (len(changed),
 177                                                               len(added),
 178                                                               len(deleted)))
 179     return added, deleted, changed
 180
 181 #exchangelib gives us days in recurrence patterns as integers,
 182 #RFC5545 wants SU,MO,TU,WE,TH,FR,SA
 183 #it has a utility function to convert to Monday, Tuesday, ...
 184 def rr_daystr_from_int(i):
 185     return exchangelib.recurrence._weekday_to_str(i).upper()[:2]
 186
 187 #for monthly patterns, we want the week (or -1 for last) combined with each
 188 #day specified
 189 def rr_daystr_monthly(p):
 190     if p.week_number == 5:
 191         wn = "-1"
 192     else:
 193         wn = str(p.week_number)
 194     return ",".join([wn + rr_daystr_from_int(x) for x in p.weekdays])
 195
 196 def rrule_from_ex(event,gcal_tz):
 197     if event.type != "RecurringMaster":
 198         logger.error("Cannot make recurrence from not-recurring event")
 199         return None
 200     if event.recurrence is None:
 201         logger.error("Empty recurrence structure")
 202         return None
 203     if isinstance(event.recurrence.pattern,
 204                   exchangelib.recurrence.DailyPattern):
 205         rr = "RRULE:FREQ=DAILY;INTERVAL=%d" % event.recurrence.pattern.interval
 206     elif isinstance(event.recurrence.pattern,
 207                     exchangelib.recurrence.WeeklyPattern):
 208         rr = "RRULE:FREQ=WEEKLY;INTERVAL=%d;BYDAY=%s;WKST=%s" % \
 209                           (event.recurrence.pattern.interval,
 210                            ",".join([rr_daystr_from_int(x) for x in event.recurrence.pattern.weekdays]),
 211                            rr_daystr_from_int(event.recurrence.pattern.first_day_of_week) )
 212     elif isinstance(event.recurrence.pattern,
 213                     exchangelib.recurrence.RelativeMonthlyPattern):
 214         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYDAY=%s" % \
 215                      (event.recurrence.pattern.interval,
 216                       rr_daystr_monthly(event.recurrence.pattern))
 217     elif isinstance(event.recurrence.pattern,
 218                     exchangelib.recurrence.AbsoluteMonthlyPattern):
 219         rr = "RRULE:FREQ=MONTHLY;INTERVAL=%d;BYMONTHDAY=%d" % \
 220                           (event.recurrence.pattern.interval,
 221                            event.recurrence.pattern.day_of_month)
 222     elif isinstance(event.recurrence.pattern,
 223                     exchangelib.recurrence.AbsoluteYearlyPattern):
 224         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYMONTHDAY=%d" % \
 225                           (event.recurrence.pattern.month,
 226                            event.recurrence.pattern.day_of_month)
 227     elif isinstance(event.recurrence.pattern,
 228                     exchangelib.recurrence.RelativeYearlyPattern):
 229         rr = "RRULE:FREQ=YEARLY;BYMONTH=%d;BYDAY=%s" % \
 230                           (event.recurrence.pattern.month,
 231                            rr_daystr_monthly(event.recurrence.pattern))
 232     else:
 233         logger.error("Recurrence %s not supported" % event.recurrence)
 234         return None
 235     if isinstance(event.recurrence.boundary,
 236                   exchangelib.recurrence.EndDatePattern):
 237         rr += ";UNTIL={0:%Y}{0:%m}{0:%d}".format(event.recurrence.boundary.end)
 238     elif isinstance(event.recurrence.boundary,
 239                     exchangelib.recurrence.NoEndPattern):
 240         pass #no end date to set
 241     else:
 242         logger.error("Recurrence %s not supported" % event.recurrence)
 243         return None
 244     return [rr]
 245
 246 def modify_recurring(ex_acct,gcal_acct,gcal_tz,
 247                      events,master,gcal_id="primary"):
 248     if master.modified_occurrences is not None:
 249         for mod in master.modified_occurrences:
 250             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 251                                                mod.original_start,
 252                                                master.is_all_day,
 253                                                master._start_timezone,
 254                                                gcal_id)
 255             if instance is None: #give up after first failure
 256                 return
 257             mod_event = get_ex_event_by_itemid(ex_acct.calendar,mod.item_id)
 258             gevent = build_gcal_event_from_ex(mod_event,gcal_tz)
 259             gevent = gcal_acct.events().update(calendarId=gcal_id,
 260                                                eventId=instance.get('id'),
 261                                                body=gevent,
 262                                                sendUpdates="none").execute()
 263             mod_event.gcal_link = gevent.get("id")
 264             mod_event.save(update_fields=["gcal_link"])
 265     if master.deleted_occurrences is not None:
 266         for d in master.deleted_occurrences:
 267             instance = get_gcal_recur_instance(gcal_acct,master.gcal_link,
 268                                                d.start,
 269                                                master.is_all_day,
 270                                                master._start_timezone,
 271                                                gcal_id)
 272             if instance is None: #give up after any failure
 273                 return
 274             if instance["status"] != "cancelled":
 275                 instance["status"]="cancelled"
 276                 gcal_acct.events().update(calendarId=gcal_id,
 277                                           eventId=instance.get('id'),
 278                                           body=instance,
 279                                           sendUpdates="none").execute()
 280
 281 def build_gcal_event_from_ex(event,gcal_tz):
 282     gevent={}
 283     gevent["summary"]=event.subject
 284     #Use the event's timezones if possible, otherwise fall back to the
 285     #target calendar timezone
 286     if event._start_timezone is not None:
 287         stz = event._start_timezone
 288     else:
 289         stz = gcal_tz
 290     if event._end_timezone is not None:
 291         etz = event._end_timezone
 292     else:
 293         etz = gcal_tz
 294     if event.is_all_day:
 295         gevent["end"]={"date": str(event.end.astimezone(etz).date())}
 296         gevent["start"]={"date": str(event.start.astimezone(stz).date())}
 297     else:
 298         gevent["end"]={"dateTime": event.end.astimezone(etz).isoformat(),
 299                        "timeZone": str(etz)}
 300         gevent["start"]={"dateTime": event.start.astimezone(stz).isoformat(),
 301                          "timeZone": str(stz)}
 302     if event.text_body is not None and event.text_body.strip() != '':
 303         gevent["description"] = event.text_body
 304     if event.location is not None:
 305         gevent["location"] = event.location
 306     gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 307     return gevent
 308
 309 def add_ex_to_gcal(ex_acct,
 310                    gcal_acct,gcal_tz,events,
 311                    added,
 312                    gcal_id="primary"):
 313     for ev_id in added:
 314         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 315         gevent = build_gcal_event_from_ex(event,gcal_tz)
 316         if event.type=="RecurringMaster":
 317             rr = rrule_from_ex(event,gcal_tz)
 318             if rr is not None:
 319                 gevent["recurrence"] = rr
 320                 print(gevent)
 321             else:
 322                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 323                 continue #don't make the gcal event
 324         gevent = gcal_acct.events().insert(calendarId=gcal_id,
 325                                            body=gevent).execute()
 326         event.gcal_link = gevent.get("id")
 327         event.save(update_fields=["gcal_link"])
 328         if event.type=="RecurringMaster" and (event.deleted_occurrences or \
 329                                               event.modified_occurrences):
 330             modify_recurring(ex_acct,gcal_acct,gcal_tz,
 331                              events,event,gcal_id)
 332             #changekey is updated by the above
 333             event.refresh()
 334         events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 335
 336 def del_ex_to_gcal(ex_acct, gcal_acct, events, deleted, gcal_id="primary"):
 337     for ev_id in deleted:
 338         if events[ev_id].gcal_link is not None:
 339             gevent = get_gcal_event_by_eventid(gcal_acct,
 340                                                events[ev_id].gcal_link,
 341                                                gcal_id)
 342             if gevent["status"] != "cancelled":
 343                 gcal_acct.events().delete(calendarId=gcal_id,
 344                                           eventId=events[ev_id].gcal_link,
 345                                           sendUpdates="none").execute()
 346
 347 def update_ex_to_gcal(ex_acct,
 348                       gcal_acct,gcal_tz,
 349                       events,changed,
 350                       gcal_id="primary"):
 351     for ev_id in changed:
 352         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 353         if event.gcal_link is None:
 354             logger.warning("Cannot apply update where event has no gcal link")
 355             continue
 356         gevent = build_gcal_event_from_ex(event,gcal_tz)
 357         if event.type=="RecurringMaster":
 358             rr = rrule_from_ex(event,gcal_tz)
 359             if rr is not None:
 360                 gevent["recurrence"] = rr
 361                 if event.deleted_occurrences or \
 362                    event.modified_occurrences:
 363                     modify_recurring(ex_acct,gcal_acct,gcal_tz,
 364                                      events,event,gcal_id)
 365                     event.refresh() #changekey is updated by the above
 366                     events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 367             else:
 368                 logger.warning("Unable to set recurrence for %s" % event.item_id)
 369                 continue #don't make the gcal event
 370         try: #may fail if we don't own the event
 371             gevent = gcal_acct.events().update(calendarId=gcal_id,
 372                                                eventId=event.gcal_link,
 373                                                body=gevent,
 374                                                sendUpdates="none").execute()
 375         except googleapiclient.errors.HttpError as err:
 376             if err.resp.status == 403:
 377                 pass
 378
 379 def match_ex_to_gcal(ex_acct,gcal_acct,gcal_tz,events,gcal_id="primary",ignore_link=True):
 380     recur = 0
 381     matched = 0
 382     skipped = 0
 383     toadd = []
 384     for ev_id in events:
 385         event = get_ex_event_by_itemid(ex_acct.calendar,ev_id)
 386         if event.gcal_link is not None and ignore_link is False:
 387             skipped += 1
 388             continue
 389         missing=True
 390         matches = gcal_acct.events().list(calendarId=gcal_id,
 391                                           timeMin=event.start.isoformat(),
 392                                           timeMax=event.end.isoformat()).execute()
 393         for ge in matches['items']:
 394             if ( ge.get("summary") is None and event.subject is None ) or \
 395                ( ge.get("summary") is not None and event.subject is not None \
 396                  and ge['summary'].strip()==event.subject.strip()):
 397                 logger.info("Matching '%s' starting at %s" % (event.subject,
 398                                                               event.start.isoformat()))
 399                 event.gcal_link = ge['id']
 400                 event.save(update_fields=["gcal_link"])
 401                 events[event.item_id] = events[event.item_id]._replace(changekey=event.changekey,gcal_link=event.gcal_link)
 402                 gevent = {}
 403                 gevent["start"] = ge["start"]
 404                 gevent["end"] = ge["end"]
 405                 gevent["extendedProperties"]={"shared": {"ex_id": event.item_id}}
 406                 try:
 407                     gcal_acct.events().update(calendarId=gcal_id,
 408                                               eventId=event.gcal_link,
 409                                               body=gevent,
 410                                               sendUpdates="none").execute()
 411                 #this may fail if we don't own the event
 412                 except googleapiclient.errors.HttpError as err:
 413                     if err.resp.status == 403:
 414                         pass
 415                 matched += 1
 416                 missing = False
 417                 break
 418         if missing == True:
 419             toadd.append(ev_id)
 420     logger.info("Matched %d events, skipped %d with existing link, and %d recurring ones" % (matched,skipped,recur))
 421     return toadd
 422
 423 def get_gcal_cred(args):
 424     #each such file can only store a single credential
 425     storage = oauth2client.file.Storage(gcal_authpath)
 426     gcal_credential = storage.get()
 427     #if no credential found, or they're invalid (e.g. expired),
 428     #then get a new one; pass --noauth_local_webserver on the command line
 429     #if you don't want it to spawn a browser
 430     if gcal_credential is None or gcal_credential.invalid:
 431         gcal_credential = oauth2client.tools.run_flow(flow,
 432                                                       storage,
 433                                                       args)
 434     return gcal_credential
 435
 436 def gcal_login(args):
 437     gcal_credential = get_gcal_cred(args)
 438     # Object to handle http requests; could add proxy details
 439     http = httplib2.Http()
 440     http = gcal_credential.authorize(http)
 441     return apiclient.discovery.build('calendar', 'v3', http=http)
 442
 443 def get_gcal_timezone(gcal_account,calendarid="primary"):
 444     gcal = gcal_account.calendars().get(calendarId=calendarid).execute()
 445     return exchangelib.EWSTimeZone.timezone(gcal['timeZone'])
 446
 447 def main():
 448     ap=argparse.ArgumentParser(description="Gooswapper calendar sync",
 449                                parents=[oauth2client.tools.argparser])
 450     ap.add_argument("exchuser",help="Exchange user e.g. 'SANGER\mv3'")
 451     ap.add_argument("exchemail",
 452                     help="Exchange calendar email e.g. ISGGroup@sanger.ac.uk")
 453     ap.add_argument("-g","--gcalid",help="google Calendar ID")
 454     ap.add_argument("-l","--loop",help="keep running indefinitely",
 455                     action="store_true")
 456     args = ap.parse_args()
 457     if args.gcalid is None:
 458         gcal_id = "primary"
 459     else:
 460         gcal_id = args.gcalid
 461
 462     #Make our config dir if it doesn't exist
 463     if not os.path.exists(gsdir):
 464         os.mkdir(gsdir,0o700)
 465     #Cache file is specific to the Exchange calendar
 466     global cachepath
 467     cachepath = gsdir + "/.cache-%s" % \
 468                 (args.exchemail.replace('@','_').replace('/','_'))
 469
 470     #log in to the accounts
 471     ex_account = ex_login(args.exchuser,args.exchemail,
 472                           gsdir+"/.gooswapper_exch_conf.dat")
 473     gcal_account = gcal_login(args)
 474     gcal_tz = get_gcal_timezone(gcal_account,gcal_id)
 475
 476     #Main loop (broken at the end if login is false)
 477     while True:
 478         try:
 479             with open(cachepath,"rb") as f:
 480                 cache = pickle.load(f)
 481         except FileNotFoundError:
 482             cache = None
 483
 484         current = get_ex_events(ex_account.calendar)
 485
 486         if cache is not None:
 487             added,deleted,changed = ex_event_changes(cache,current)
 488             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 489                            added,gcal_id)
 490             #delete op needs the "cache" set, as that has the link ids in
 491             #for events that are now deleted
 492             del_ex_to_gcal(ex_account,gcal_account,cache,deleted,gcal_id)
 493             update_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 494                               changed,gcal_id)
 495         else:
 496             toadd = match_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 497                                      gcal_id)
 498             add_ex_to_gcal(ex_account,gcal_account,gcal_tz,current,
 499                            toadd,gcal_id)
 500
 501         with open(cachepath,"wb") as f:
 502             pickle.dump(current,f)
 503
 504         #If not looping, break here (after 1 run)
 505         if args.loop==False:
 506             break
 507         #otherwise, wait 10 minutes, then go round again
 508         time.sleep(600)
 509
 510 if __name__ == "__main__":
 511     main()
 512
 513