src/net/bionicmessage/objects/MultipleSourceICalendarObjectStore.java

   1 /*
   2  * ICalendarObjectStore.java
   3  *
   4  * Created on 26 August 2006, 14:56
   5  * Copyright (c) 2006 Mathew McBride / "BionicMessage.net"
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy of this
   7  * software and associated documentation files (the "Software"), to deal in the Software
   8  * without restriction, including without limitation the rights to use, copy, modify, merge,
   9  * publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
  10  * to whom the Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be included in all copies or
  13  * substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
  16  * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
  17  * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
  18  * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  19  * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  *
  22  *
  23  */
  24
  25 package net.bionicmessage.objects;
  26
  27 import java.io.File;
  28 import java.io.FileInputStream;
  29 import java.io.IOException;
  30 import java.io.PrintWriter;
  31 import java.io.StringReader;
  32 import java.io.StringWriter;
  33 import java.sql.SQLException;
  34 import java.util.ArrayList;
  35 import java.util.Hashtable;
  36 import java.util.Iterator;
  37 import java.util.List;
  38 import java.util.Map;
  39 import java.util.Properties;
  40 import java.util.logging.FileHandler;
  41 import net.bionicmessage.groupdav.*;
  42 import net.fortuna.ical4j.data.CalendarBuilder;
  43 import net.fortuna.ical4j.model.Calendar;
  44 import net.fortuna.ical4j.model.Component;
  45 import net.fortuna.ical4j.model.property.Summary;
  46 import net.fortuna.ical4j.model.property.Uid;
  47 import java.util.logging.*;
  48 import net.bionicmessage.utils.HTMLFormatter;
  49 import net.fortuna.ical4j.model.property.DtEnd;
  50 import net.fortuna.ical4j.model.property.DtStart;
  51 import net.bionicmessage.pdi.*;
  52
  53 /**
  54  * ICalendarObjectStore builds on ObjectTracking to maintain a client relationship
  55  * with a GroupDAV server.
  56  * @author matt
  57  */
  58 public class MultipleSourceICalendarObjectStore {
  59
  60     groupDAV client = null;
  61     public static int OPTION_SSL = 1; /* Are we using SSL? */
  62     public static int OPTION_I_AM_AN_EXPERT = 128; /* Allows the user to do dangerous things,
  63      * such as changing the URL the store points to
  64      * after init */
  65     public static int OPTION_PURGE = 64; /* Purge the store contents, i.e for Funambol
  66      * Slow sync */
  67     public static int OPTION_TODO = 32;
  68     public static int OPTION_NEWHANDLER = 16;
  69     /** Retrieve password from store config file */
  70     public static int OPTION_STOREDPASS = 8;
  71
  72     /** Use a single log file */
  73     public static int OPTION_SINGLELOG = 4;
  74
  75     /** Base property for store location */
  76     public static final String PROPERTY_STORE_LOCATION = "store.location";
  77     /** Properties name for server */
  78     public static final String PROPERTY_SERVER = "store.server";
  79     /** Properties name for user */
  80     public static final String PROPERTY_USER = "store.user";
  81     /** Properties name for password */
  82     public static final String PROPERTY_PASSWORD = "store.password";
  83     /** Base property for sources */
  84     public static final String BASE_PROPERTY_SOURCE = "store.source.";
  85
  86     boolean sslmode = false;
  87     boolean expertmode = false;
  88     boolean purgemode = false;
  89     boolean todomode = false;
  90     boolean newhandler = false;
  91     boolean singlelog = false;
  92     MultipleSourceObjectTracking obtrack = null;
  93     String obtrackdir = "";
  94     Properties props = null;
  95     File propfile = null;
  96     /* Queues for reports */
  97     ArrayList addedToStore = new ArrayList();
  98     ArrayList updatedInStore = new ArrayList();
  99     ArrayList deletedFromStore = new ArrayList();
 100     ArrayList untouched = new ArrayList();
 101     ArrayList addedToServer = new ArrayList();
 102     ArrayList updatedOnServer = new ArrayList();
 103     ArrayList deletedFromServer = new ArrayList();
 104     /* Object cache */
 105     Hashtable objCache = new Hashtable();
 106
 107     /* List of source defined */
 108     Hashtable<String, String> sources = new Hashtable();
 109     /* Logger */
 110     Logger log = Logger.getLogger("groupdav.icalobjectstore");
 111     FileHandler fh = null;
 112
 113     /** Creates a new instance of VCalendarObjectStore */
 114     public MultipleSourceICalendarObjectStore(String storedir, int options) {
 115         int ssltest = options & MultipleSourceICalendarObjectStore.OPTION_SSL;
 116         if (ssltest == OPTION_SSL) {
 117             sslmode = true;
 118         }
 119         int experttest = options & OPTION_I_AM_AN_EXPERT;
 120         if (experttest == OPTION_I_AM_AN_EXPERT) {
 121             expertmode = true;
 122         }
 123         int purgetest = options & OPTION_PURGE;
 124         if (purgetest == OPTION_PURGE) {
 125             purgemode = true;
 126         }
 127         int todotest = options & OPTION_TODO;
 128         if (todotest == OPTION_TODO) {
 129             todomode = true;
 130         }
 131         int newtest = options & OPTION_NEWHANDLER;
 132         if (newtest == OPTION_NEWHANDLER) {
 133             newhandler = true;
 134         }
 135         int singletest = options & OPTION_SINGLELOG;
 136         if (singletest == OPTION_SINGLELOG) {
 137             singlelog = true;
 138         }
 139         obtrackdir = storedir + File.separatorChar + "obtrack" + File.separatorChar;
 140         if (purgemode) {
 141             File obtrackdirf = new File(obtrackdir);
 142             String[] files = obtrackdirf.list();
 143             if (obtrackdirf.exists() && files.length > 0) {
 144                 for (int i = 0; i < files.length; i++) {
 145                     String string = files[i];
 146                     File toDelete = new File(obtrackdir + File.separatorChar + string);
 147                     boolean deleted = toDelete.delete();
 148                     System.out.println("Delete " + toDelete.getAbsolutePath() + "..." + deleted);
 149                 }
 150             }
 151             obtrackdirf.delete();
 152         }
 153         obtrack = new MultipleSourceObjectTracking(obtrackdir);
 154         props = new Properties();
 155         propfile = new File(storedir + File.separatorChar + "vcalstoreprops");
 156         if (propfile.exists()) {
 157             try {
 158                 FileInputStream propstream = new FileInputStream(propfile);
 159                 props.load(propstream);
 160             } catch (IOException ex) {
 161                 ex.printStackTrace();
 162             }
 163         }
 164         try {
 165             long curTime = new java.util.Date().getTime();
 166             HTMLFormatter hf = new HTMLFormatter();
 167             if (singlelog) {
 168                 fh = new FileHandler(storedir + File.separatorChar + "storelog.html");
 169             } else {
 170                 fh = new FileHandler(storedir + File.separatorChar + "storelog-" + curTime + ".html");
 171             }
 172             fh.setFormatter(hf);
 173             log.addHandler(fh);
 174             fh.setLevel(Level.ALL);
 175             log.setLevel(Level.ALL);
 176         } catch (SecurityException ex) {
 177             ex.printStackTrace();
 178         } catch (IOException ex) {
 179             ex.printStackTrace();
 180         }
 181     }
 182
 183     public void loadSourcesFromProps() {
 184         Iterator listOfProps = props.keySet().iterator();
 185         while (listOfProps.hasNext()) {
 186             String key = (String) listOfProps.next();
 187             if (key.contains(BASE_PROPERTY_SOURCE)) {
 188                 String loc = props.getProperty(key);
 189                 key = key.replace(BASE_PROPERTY_SOURCE, "");
 190                 setStore(key, loc);
 191             }
 192         }
 193     }
 194     public void setProperties(Properties p) {
 195         props = p;
 196     }
 197     public void setServer(String url) {
 198         if (props.getProperty(PROPERTY_SERVER) == null || expertmode) {
 199             props.setProperty(PROPERTY_SERVER, url);
 200         }
 201     }
 202
 203     public void setStore(String name, String storeurl) {
 204         if (!storeurl.substring(storeurl.length()).equals("/")) {
 205             storeurl = storeurl + "/";
 206         }
 207         if (props.getProperty(BASE_PROPERTY_SOURCE + name) == null || expertmode) {
 208             props.setProperty(BASE_PROPERTY_SOURCE + name, storeurl);
 209         }
 210         sources.put(name, storeurl);
 211     }
 212
 213     public void connect_Base64(String b64cache) throws Exception {
 214         client = new groupDAV(props.getProperty(PROPERTY_SERVER), b64cache);
 215         afterConnect();
 216     }
 217
 218     public void connect_plain(String user, String pass) throws Exception {
 219         client = new groupDAV(props.getProperty(PROPERTY_SERVER), user, pass);
 220         afterConnect();
 221     }
 222
 223     public void connect_storedpass() throws Exception {
 224         client = new groupDAV(props.getProperty(PROPERTY_SERVER), props.getProperty(PROPERTY_USER), props.getProperty(PROPERTY_PASSWORD));
 225         afterConnect();
 226     }
 227
 228     private void afterConnect() throws Exception {
 229         client.setLogger(log);
 230     }
 231
 232     public Hashtable getStores() {
 233         return client.getURLLocs();
 234     }
 235
 236     /** Processing command for objects which have been added on the server
 237      *  and need to be added here
 238      */
 239     private String addFromServerToStore(String storeName, String url) throws Exception {
 240         return addFromServerToStore(storeName, url, addedToStore);
 241     }
 242
 243     private String addFromServerToStore(String storeName, String url, List logList) throws Exception {
 244         GroupDAVObject obj = client.getObject(url);
 245         String name = "";
 246         Calendar cd = constructCalendarObject(obj.getContent());
 247         Component vcal = (Component) cd.getComponents().get(0);
 248         Uid uid = (Uid) vcal.getProperty("UID");
 249         objCache.put(uid.getValue(), cd);
 250         Summary summ = (Summary) vcal.getProperty("SUMMARY");
 251         long dtstart = 0;
 252         long dtend = 0;
 253         if (!todomode) {
 254             DtStart dts = (DtStart) vcal.getProperty("DTSTART");
 255             DtEnd dte = (DtEnd) vcal.getProperty("DTEND");
 256             java.util.Date dts_date = (java.util.Date) dts.getDate();
 257             java.util.Date dte_date = (java.util.Date) dte.getDate();
 258             dtstart = dts_date.getTime() / 1000;
 259             dtend = dte_date.getTime() / 1000;
 260             dts = null;
 261             dte = null;
 262         }
 263         obtrack.createObject(storeName, uid.getValue(), url, obj.getEtag(), summ.getValue().trim(), obj.getContent(), dtstart, dtend);
 264         logList.add(uid.getValue());
 265         return uid.getValue();
 266     }
 267
 268     private void updateObjectFromServer(String uid) throws Exception {
 269         GroupDAVObject obj = client.getObject(obtrack.getObjectURL(uid));
 270         Calendar cd = constructCalendarObject(obj.getContent());
 271         Component vcal = (Component) cd.getComponents().get(0);
 272         Summary summ = (Summary) vcal.getProperty("SUMMARY");
 273         long dtstart = 0;
 274         long dtend = 0;
 275         if (!todomode) {
 276             DtStart dts = (DtStart) vcal.getProperty("DTSTART");
 277             DtEnd dte = (DtEnd) vcal.getProperty("DTEND");
 278             dtstart = dts.getDate().getTime();
 279             dtend = dts.getDate().getTime();
 280             dts = null;
 281             dte = null;
 282         }
 283         obtrack.updateEtag(uid, obj.getEtag());
 284         obtrack.updateObject(uid, cd.toString());
 285         obtrack.updateName(uid, summ.getValue());
 286         obtrack.updateDate(uid, dtstart, dtend);
 287         updatedInStore.add(uid);
 288     }
 289
 290     private void deleteFromStore(String uid) throws Exception {
 291         obtrack.deleteObject(uid);
 292         deletedFromStore.add(uid);
 293     }
 294
 295     /* -> End store/server side */
 296     /* Client->Store->Server side */
 297     /** Adds an object to the server.
 298      * @param sourceName the name of the source to add to
 299      * @param uid The UID of the object to add
 300      * @param name Name of the object to be added
 301      * @param contents The ical2 data of the object
 302      * @param dtstart The DTSTART of the object in Unix time
 303      * @param dtend The DTEND of the object in Unix time
 304      * @return Server UID of object added.
 305      */
 306     public String addObject(String sourceName, String uid, String name, String contents, long dtstart, long dtend) throws Exception {
 307         String preexistingUid = obtrack.findObjectByName(name.trim());
 308         /* Often after client restarts we might get an object we already have
 309          * but with a different UID. We should treat these as merge ops */
 310         if (preexistingUid != null && preexistingUid.length() > 1) {
 311             // Check to see if the dtstart|dtend are the same
 312             if (!todomode) {
 313                 long dtustart = obtrack.getDtStartForUid(preexistingUid);
 314                 long dtuend = obtrack.getDtEndFromUid(preexistingUid);
 315                 /* Should no longer be needed thanks to getSyncItemKeysFromTwin() */
 316                 /*     if (dtustart == dtstart && dtuend == dtend) {
 317                 // this object is a duplicate of an existing timeslot. merge it
 318                 log.info("Merging " +  preexistingUid + " and " + uid);
 319                 mergeObject(preexistingUid,contents,obtrack.getObjectContents(preexistingUid));
 320                 return 2;
 321                 } */
 322             } else {
 323                 // log.info("Merging todos " + preexistingUid + " and " + uid);
 324                 // Todos are always merges
 325                 //mergeObject(preexistingUid,contents,obtrack.getObjectContents(preexistingUid));
 326                 // return 2;
 327             }
 328         }
 329         /* Post to the server first so we don't have objects in the store
 330          * which the server does not want */
 331         GroupDAVObject gbo = client.postObject(sources.get(sourceName), uid, contents);
 332         /* Retrieve the object again since the server would've had an opportunity
 333          * to manipulate it, i.e change UID */
 334         String srvuid = addFromServerToStore(sourceName, gbo.getLocation(), addedToServer);
 335         return srvuid;
 336     }
 337
 338     /**
 339      * Replace a pre-existing object in the store with a new one.
 340      * @param uid The UID of the object to replace
 341      * @param name If the object's name has changed, the store will be updated with the new name
 342      *
 343      * If name is null, the name will not be modified.
 344      * @param contents Contents of the object replacing the object
 345      * @param dtstart The start time of the object. If dtstart and dtend are 0, dates will not be changed
 346      * @param dtend The end time of the object. If dtstart and dtend are 0, dates will not be changed
 347      */
 348     public int replaceObject(String storeName, String uid, String name, String contents, long dtstart, long dtend) throws Exception {
 349         String oldetag = obtrack.getObjectEtag(uid);
 350         GroupDAVObject gbo = client.modifyObject(obtrack.getObjectURL(uid), contents, oldetag);
 351         obtrack.deleteObject(uid);
 352         addFromServerToStore(storeName, gbo.getLocation(), updatedOnServer);
 353         return 0;
 354     }
 355
 356     public String searchUids(String name, long dstime, long detime) throws Exception {
 357         // Find potential names
 358         @SuppressWarnings("unchecked")
 359         ArrayList<String> potentialNames = obtrack.findObjectsByName(name);
 360         for (int i = 0; i < potentialNames.size(); i++) {
 361             String id = potentialNames.get(i);
 362             // Do we have a match?
 363             boolean match = obtrack.doesUidMatchTimes(id, dstime, detime);
 364             if (match) {
 365                 return id;
 366             }
 367         }
 368         return null;
 369     }
 370
 371     public void deleteObject(String uid) throws Exception {
 372         String oldetag = obtrack.getObjectEtag(uid);
 373         GroupDAVObject gbo = client.deleteObject(obtrack.getObjectURL(uid), oldetag);
 374         obtrack.deleteObject(uid);
 375         deletedFromServer.add(uid);
 376     }
 377
 378     public Calendar getObjectFromStore(String uid) throws Exception {
 379         if (objCache.get(uid) != null) {
 380             log.fine("Object " + uid + "has been retrieved from store");
 381             Calendar cal = (Calendar) objCache.get(uid);
 382             return cal;
 383         } else {
 384             if (!obtrack.doesObjectExist(uid)) {
 385                 throw new Exception("We do not have an object for " + uid);
 386             } else {
 387                 log.fine("We are constructing an object for " + uid);
 388                 return constructCalendarObject(obtrack.getObjectContents(uid));
 389             }
 390         }
 391     }
 392
 393     /**
 394      *
 395      * @param content
 396      * @throws java.lang.Exception
 397      * @return The UID of the object
 398      */
 399     private Calendar constructCalendarObject(String content) throws Exception {
 400         // Fix issue with blank ORGANIZER:MAILTO: from Kontact
 401         String ct = content.replace("ORGANIZER:MAILTO:\r\n", "");
 402         // Fix issue with stupid \n output behavior in ZideStore 1.4
 403         ct = ct.replace("\r\\n", "\r\n \\n");
 404         // ct = ct.replace(" \r\n", "\r\n  "); - Disabled
 405         // Don't remove - used for debugger sessions
 406         String debugct = content.replace("\r", "xR").replace("\n", "xN");
 407         CalendarBuilder cbuild = new CalendarBuilder();
 408         StringReader sread = new StringReader(ct);
 409         Calendar cd = cbuild.build(sread);
 410         return cd;
 411     }
 412
 413     public void startSync() throws Exception {
 414         log.info("Sync started....");
 415         obtrack.init();
 416         Iterator<String> sourceList = sources.keySet().iterator();
 417         while (sourceList.hasNext()) {
 418             String sourceName = sourceList.next();
 419             String sourceLoc = sources.get(sourceName);
 420             List<String> objectsForSource = client.listObjects(sourceLoc);
 421             Map etags = client.getEtags();
 422             for (int i = 0; i < objectsForSource.size(); i++) {
 423                 String url = (String) objectsForSource.get(i);
 424                 // Do we have the object?
 425                 boolean haveurl = obtrack.doesURLExist(url);
 426                 if (haveurl) {
 427                     String stuid = obtrack.findObjectByURL(url);
 428                     String stetag = obtrack.getObjectEtag(stuid);
 429                     String getag = (String) etags.get(url);
 430                     if (!stetag.equals(getag)) {
 431                         updateObjectFromServer(stuid);
 432                     } else {
 433                         untouched.add(stuid);
 434                     }
 435                     log.fine("We have the URL: " + url);
 436                 } else {
 437                     try {
 438                         addFromServerToStore(sourceName, url);
 439                     } catch (Exception ex) {
 440                         log.throwing(this.getClass().getName(), "startSync", ex);
 441                         ex.printStackTrace();
 442                     }
 443                     log.fine("We added the URL: " + url);
 444                 }
 445             }
 446             ArrayList URLarray = obtrack.getURLListForSource(sourceName);
 447             for (int i = 0; i < URLarray.size(); i++) {
 448                 String url = (String) URLarray.get(i);
 449                 if (null == etags.get(url)) {
 450                     log.fine("Deleted on server: " + url);
 451                     deleteFromStore(obtrack.findObjectByURL(url));
 452                 }
 453             }
 454         }
 455     }
 456
 457     public ArrayList getUnModifiedUIDS() {
 458         return untouched;
 459     }
 460
 461     public ArrayList getAddedToServerUIDS() {
 462         return addedToServer;
 463     }
 464
 465     public ArrayList getAddedToStoreUIDS() {
 466         return addedToStore;
 467     }
 468
 469     public ArrayList getDeletedFromServerUIDS() {
 470         return deletedFromServer;
 471     }
 472
 473     public ArrayList getDeletedFromStoreUIDS() {
 474         return deletedFromStore;
 475     }
 476
 477     public ArrayList getUpdatedInStoreUIDS() {
 478         return updatedInStore;
 479     }
 480
 481     public ArrayList getUpdatedOnServerUIDS() {
 482         return updatedOnServer;
 483     }
 484
 485     public ArrayList getUIDList() throws SQLException {
 486         return obtrack.getUIDList();
 487     }
 488
 489     public void printDebugReport() throws Exception {
 490         StringWriter sw = new StringWriter();
 491         PrintWriter lw = new PrintWriter(sw);
 492         ArrayList uids = obtrack.getUIDList();
 493         for (int i = 0; i < uids.size(); i++) {
 494             String uid = (String) uids.get(i);
 495             String url = obtrack.getObjectURL(uid);
 496             if (url == null) {
 497                 throw new Exception("URL for " + uid + " is null");
 498             }
 499             String etag = obtrack.getObjectEtag(uid);
 500             String name = obtrack.getObjectName(uid);
 501             String data = obtrack.getObjectContents(uid);
 502             long dtstart = 0;
 503             long dtend = 0;
 504             if (!todomode) {
 505                 dtstart = obtrack.getDtStartForUid(uid);
 506                 dtend = obtrack.getDtEndFromUid(uid);
 507             }
 508             lw.println("----------");
 509             lw.println("UID:" + uid);
 510             lw.println("URL:" + url);
 511             lw.println("ETAG:" + etag);
 512             lw.println("NAME:" + name);
 513             if (!todomode) {
 514                 lw.println("DTSTART:" + dtstart);
 515                 lw.println("DTEND:" + dtend);
 516             }
 517             lw.println("DATA FOLLOWS:");
 518             lw.println(data);
 519             lw.println("----------");
 520         }
 521         lw.println("Objects added to store: ");
 522         for (int i = 0; i < addedToStore.size(); i++) {
 523             String uid = (String) addedToStore.get(i);
 524             lw.println("A:    " + uid);
 525         }
 526         lw.println("Objects updated from server: ");
 527         for (int i = 0; i < updatedInStore.size(); i++) {
 528             String uid = (String) updatedInStore.get(i);
 529             lw.println("U:   " + uid);
 530         }
 531         lw.println("Objects deleted from store: ");
 532         for (int i = 0; i < deletedFromStore.size(); i++) {
 533             String uid = (String) deletedFromStore.get(i);
 534             lw.println("D:    " + uid);
 535         }
 536         lw.println("Objects added to the server: ");
 537         for (int i = 0; i < addedToServer.size(); i++) {
 538             String uid = (String) addedToServer.get(i);
 539             lw.println("SA:   " + uid);
 540         }
 541         lw.println("Objects merged to server: ");
 542         for (int i = 0; i < updatedOnServer.size(); i++) {
 543             String uid = (String) updatedOnServer.get(i);
 544             lw.println("SM:   " + uid);
 545         }
 546         lw.println("Objects deleted from server: ");
 547         for (int i = 0; i < deletedFromServer.size(); i++) {
 548             String uid = (String) deletedFromServer.get(i);
 549             lw.println("SD:   " + uid);
 550         }
 551         lw.flush();
 552         lw.close();
 553         log.info(sw.toString());
 554     }
 555
 556     public void close() throws Exception {
 557         fh.close();
 558         obtrack.terminate();
 559     }
 560
 561     public static void main(String[] args) {
 562         String storeloc = "";
 563         if (args.length > 0 && args[0] != null) {
 564             storeloc = args[0];
 565         } else {
 566             storeloc = System.getProperty("user.home") + System.getProperty("file.separator") + "icalobjectstoretest";
 567         }
 568         MultipleSourceICalendarObjectStore vco = new MultipleSourceICalendarObjectStore(storeloc, MultipleSourceICalendarObjectStore.OPTION_STOREDPASS + MultipleSourceICalendarObjectStore.OPTION_TODO);
 569
 570         try {
 571             vco.connect_storedpass();
 572             vco.startSync();
 573             vco.printDebugReport();
 574         } catch (Exception ex) {
 575             ex.printStackTrace();
 576         }
 577     }
 578 }