[client][managed][offline] Implement true upsert

Author: hasankhan

sdk/Managed/src/Microsoft.WindowsAzure.MobileServices.SQLiteStore/MobileServiceSQLiteStore.cs

@@ -25,7 +25,7 @@ public class MobileServiceSQLiteStore : MobileServiceLocalStore
         /// Note: The default maximum number of parameters allowed by sqlite is 999
         /// See: http://www.sqlite.org/limits.html#max_variable_number
         /// </summary>
-        private const int MaxParametersPerUpsertQuery = 800;
+        private const int MaxParametersPerQuery = 800;
 
         private Dictionary<string, TableDefinition> tableMap = new Dictionary<string, TableDefinition>(StringComparer.OrdinalIgnoreCase);
         private SQLiteConnection connection;
@@ -183,64 +183,17 @@ private Task UpsertAsyncInternal(string tableName, IEnumerable<JObject> items, b
                 return Task.FromResult(0);
             }
 
-            // Generate the prepared insert statement
-            string sqlBase = String.Format(
-                "INSERT OR REPLACE INTO {0} ({1}) VALUES ",
-                SqlHelpers.FormatTableName(tableName),
-                String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember))
-            );
-
-            // Use int division to calculate how many times this record will fit into our parameter quota
-            int batchSize = MaxParametersPerUpsertQuery / columns.Count;
-            if (batchSize == 0)
-            {
-                throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerUpsertQuery));
-            }
 
-            foreach (var batch in items.Split(maxLength: batchSize))
-            {
-                var sql = new StringBuilder(sqlBase);
-                var parameters = new Dictionary<string, object>();
+            this.ExecuteNonQuery("BEGIN TRANSACTION", null);
 
-                foreach (JObject item in batch)
-                {
-                    AppendInsertValuesSql(sql, parameters, columns, item);
-                    sql.Append(",");
-                }
+            BatchInsert(tableName, items, columns.Where(c => c.Name.Equals(MobileServiceSystemColumns.Id)).Take(1).ToList());
+            BatchUpdate(tableName, items, columns);
 
-                if (parameters.Any())
-                {
-                    sql.Remove(sql.Length - 1, 1); // remove the trailing comma
-                    this.ExecuteNonQuery(sql.ToString(), parameters);
-                }
-            }
+            this.ExecuteNonQuery("COMMIT TRANSACTION", null);
 
             return Task.FromResult(0);
         }
 
-        private static void AppendInsertValuesSql(StringBuilder sql, Dictionary<string, object> parameters, List<ColumnDefinition> columns, JObject item)
-        {
-            sql.Append("(");
-            int colCount = 0;
-            foreach (var column in columns)
-            {
-                if (colCount > 0)
-                    sql.Append(",");
-
-                JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase);
-                object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType);
-
-                //The paramname for this field must be unique within this statement
-                string paramName = "@p" + parameters.Count;
-
-                sql.Append(paramName);
-                parameters[paramName] = value;
-
-                colCount++;
-            }
-            sql.Append(")");
-        }
-
         /// <summary>
         /// Deletes items from local table that match the given query.
         /// </summary>
@@ -373,6 +326,110 @@ private void CreateAllTables()
             }
         }
 
+        private void BatchUpdate(string tableName, IEnumerable<JObject> items, List<ColumnDefinition> columns)
+        {
+            if (columns.Count <= 1)
+            {
+                return; // For update to work there has to be at least once column besides Id that needs to be updated
+            }
+
+            ValidateParameterCount(columns.Count);
+
+            string sqlBase = String.Format("UPDATE {0} SET ", SqlHelpers.FormatTableName(tableName));
+
+            foreach (JObject item in items)
+            {
+                var sql = new StringBuilder(sqlBase);
+                var parameters = new Dictionary<string, object>();
+
+                ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id));
+                if (idColumn == null)
+                {
+                    continue;
+                }
+
+                foreach (var column in columns.Where(c => c != idColumn))
+                {
+                    string paramName = AddParameter(item, parameters, column);
+
+                    sql.AppendFormat("{0} = {1}", SqlHelpers.FormatMember(column.Name), paramName);
+                    sql.Append(",");
+                }
+
+                if (parameters.Any())
+                {
+                    sql.Remove(sql.Length - 1, 1); // remove the trailing comma
+
+                }
+
+                sql.AppendFormat(" WHERE {0} = {1}", SqlHelpers.FormatMember(MobileServiceSystemColumns.Id), AddParameter(item, parameters, idColumn));
+
+                this.ExecuteNonQuery(sql.ToString(), parameters);
+            }
+        }
+
+        private void BatchInsert(string tableName, IEnumerable<JObject> items, List<ColumnDefinition> columns)
+        {
+            if (columns.Count == 0) // we need to have some columns to insert the item
+            {
+                return;
+            }
+
+            // Generate the prepared insert statement
+            string sqlBase = String.Format(
+                "INSERT OR IGNORE INTO {0} ({1}) VALUES ",
+                SqlHelpers.FormatTableName(tableName),
+                String.Join(", ", columns.Select(c => c.Name).Select(SqlHelpers.FormatMember))
+            );
+
+            // Use int division to calculate how many times this record will fit into our parameter quota
+            int batchSize = ValidateParameterCount(columns.Count);
+
+            foreach (var batch in items.Split(maxLength: batchSize))
+            {
+                var sql = new StringBuilder(sqlBase);
+                var parameters = new Dictionary<string, object>();
+
+                foreach (JObject item in batch)
+                {
+                    AppendInsertValuesSql(sql, parameters, columns, item);
+                    sql.Append(",");
+                }
+
+                if (parameters.Any())
+                {
+                    sql.Remove(sql.Length - 1, 1); // remove the trailing comma
+                    this.ExecuteNonQuery(sql.ToString(), parameters);
+                }
+            }
+        }
+
+        private static int ValidateParameterCount(int parametersCount)
+        {
+            int batchSize = MaxParametersPerQuery / parametersCount;
+            if (batchSize == 0)
+            {
+                throw new InvalidOperationException(string.Format(Properties.Resources.SQLiteStore_TooManyColumns, MaxParametersPerQuery));
+            }
+            return batchSize;
+        }
+
+        private static void AppendInsertValuesSql(StringBuilder sql, Dictionary<string, object> parameters, List<ColumnDefinition> columns, JObject item)
+        {
+            sql.Append("(");
+            int colCount = 0;
+            foreach (var column in columns)
+            {
+                if (colCount > 0)
+                    sql.Append(",");
+
+                sql.Append(AddParameter(item, parameters, column));
+
+                colCount++;
+            }
+            sql.Append(")");
+        }
+
         internal virtual void CreateTableFromObject(string tableName, IEnumerable<ColumnDefinition> columns)
         {
             ColumnDefinition idColumn = columns.FirstOrDefault(c => c.Name.Equals(MobileServiceSystemColumns.Id));
@@ -404,6 +461,21 @@ internal virtual void CreateTableFromObject(string tableName, IEnumerable<Column
             // NOTE: In SQLite you cannot drop columns, only add them.
         }
 
+        private static string AddParameter(JObject item, Dictionary<string, object> parameters, ColumnDefinition column)
+        {
+            JToken rawValue = item.GetValue(column.Name, StringComparison.OrdinalIgnoreCase);
+            object value = SqlHelpers.SerializeValue(rawValue, column.StoreType, column.JsonType);
+            string paramName = CreateParameter(parameters, value);
+            return paramName;
+        }
+
+        private static string CreateParameter(Dictionary<string, object> parameters, object value)
+        {
+            string paramName = "@p" + parameters.Count;
+            parameters[paramName] = value;
+            return paramName;
+        }
+
         /// <summary>
         /// Executes a sql statement on a given table in local SQLite database.
         /// </summary>

sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.Query.cs

@@ -274,13 +274,19 @@ private static async Task<MobileServiceSQLiteStore> SetupMathTestTable(JObject[]
             var store = new MobileServiceSQLiteStore(TestDbName);
             store.DefineTable(MathTestTable, new JObject()
             {
+                { "id", String.Empty },
                 { "val", 0f },
                 { "expected", 0f }
             });
 
             await store.InitializeAsync();
 
-            await InsertAll(store, MathTestTable, mathTestData);
+            foreach (JObject item in mathTestData)
+            {
+                item[MobileServiceSystemColumns.Id] = Guid.NewGuid().ToString();
+            }
+
+            await store.UpsertAsync(MathTestTable, mathTestData, fromServer: false);
 
             return store;
         }
@@ -308,7 +314,7 @@ private static async Task<MobileServiceSQLiteStore> SetupTestTable()
 
             if (!queryTableInitialized)
             {
-                await InsertAll(store, TestTable, testData);
+                await store.UpsertAsync(TestTable, testData, fromServer: false);
             }
 
             queryTableInitialized = true;
@@ -349,13 +355,5 @@ private static async Task<T> Query<T>(MobileServiceSQLiteStore store, string tab
         {
             return (T)await store.ReadAsync(MobileServiceTableQueryDescription.Parse(tableName, query));
         }
-
-        private async static Task InsertAll(MobileServiceSQLiteStore store, string tableName, JObject[] items)
-        {
-            foreach (JObject item in items)
-            {
-                await store.UpsertAsync(tableName, new[] { item }, fromServer: false);
-            }
-        }
     }
 }

sdk/Managed/test/Microsoft.WindowsAzure.MobileServices.SQLiteStore.Test/UnitTests/SQLiteStoreTests.cs

@@ -24,6 +24,8 @@ public class SQLiteStoreTests : TestBase
         [AsyncTestMethod]
         public async Task InitializeAsync_InitializesTheStore()
         {
+            TestUtilities.DropTestTable(TestDbName, TestTable);
+
             var store = new MobileServiceSQLiteStore(TestDbName);
             store.DefineTable(TestTable, new JObject()
             {
@@ -221,6 +223,8 @@ public void UpsertAsync_Throws_WhenStoreIsNotInitialized()
         [AsyncTestMethod]
         public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal()
         {
+            TestUtilities.DropTestTable(TestDbName, TestTable);
+
             using (var store = new MobileServiceSQLiteStore(TestDbName))
             {
                 store.DefineTable(TestTable, new JObject()
@@ -240,6 +244,8 @@ public async Task UpsertAsync_Throws_WhenColumnInItemIsNotDefinedAndItIsLocal()
         [AsyncTestMethod]
         public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFromServer()
         {
+            TestUtilities.DropTestTable(TestDbName, TestTable);
+
             using (var store = new MobileServiceSQLiteStore(TestDbName))
             {
                 store.DefineTable(TestTable, new JObject()
@@ -250,13 +256,20 @@ public async Task UpsertAsync_DoesNotThrow_WhenColumnInItemIsNotDefinedAndItIsFr
 
                 await store.InitializeAsync();
 
-                await store.UpsertAsync(TestTable, new[] { new JObject() { { "notDefined", "okok" }, { "dob", DateTime.UtcNow } } }, fromServer: true);
+                await store.UpsertAsync(TestTable, new[] { new JObject() 
+                { 
+                    { "id", "abc" },
+                    { "notDefined", "okok" }, 
+                    { "dob", DateTime.UtcNow } 
+                } }, fromServer: true);
             }
         }
 
         [AsyncTestMethod]
         public async Task UpsertAsync_DoesNotThrow_WhenItemIsEmpty()
         {
+            TestUtilities.DropTestTable(TestDbName, TestTable);
+
             using (var store = new MobileServiceSQLiteStore(TestDbName))
             {
                 store.DefineTable(TestTable, new JObject()
@@ -347,6 +360,7 @@ public async Task UpsertAsync_UpdatesTheRow_WhenItExists()
                 await store.UpsertAsync(TestTable, new[]{new JObject() 
                 { 
                     { "id", "abc" }, 
+                    { "text", "xyz" },
                     { "__createdAt", DateTime.Now } 
                 }}, fromServer: false);
 
@@ -355,6 +369,12 @@ await store.UpsertAsync(TestTable, new[]{new JObject()
                     { "id", "abc" }, 
                     { "__createdAt", new DateTime(200,1,1) } 
                 }}, fromServer: false);
+
+                JObject result = await store.LookupAsync(TestTable, "abc");
+
+                Assert.AreEqual(result.Value<string>("id"), "abc");
+                Assert.AreEqual(result.Value<string>("text"), "xyz");
+                Assert.AreEqual(result.Value<string>("__createdAt"), "01/01/0200 00:00:00");
             }
             long count = TestUtilities.CountRows(TestDbName, TestTable);
             Assert.AreEqual(count, 1L);
@@ -532,8 +552,9 @@ public static void DefineTestTable(MobileServiceSQLiteStore store)
         {
             store.DefineTable(TestTable, new JObject()
             {
-                {"id", String.Empty },
-                {"__createdAt", DateTime.Now}
+                { "id", String.Empty },
+                { "text", String.Empty },
+                { "__createdAt", DateTime.Now }
             });
         }
     }