Fix SQLRDD: reconnect handling, paging/EOF, delete, and write-flush data-integrity bugs

src/Runtime/XSharp.SQLRdd/Classes/Command.prg

@@ -103,6 +103,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable
     /// <seealso cref="SqlDbConnection.LastException"/>
     method GetSchemaTable() as DataTable
         try
+            self:Connection:ForceOpen()
             if ! SELF:_TryBindParameters()
                 return null
             endif
@@ -150,6 +151,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable
     /// <seealso cref="SqlDbConnection.LastException"/>
     method ExecuteScalar(cTable := "" as string) as object
         try
+            self:Connection:ForceOpen()
             if String.IsNullOrEmpty(cTable)
                 cTable := SELF:Name
             endif
@@ -176,6 +178,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable
     /// <seealso cref="SqlDbConnection.LastException"/>
     method ExecuteReader(cTable := "" as string) as DbDataReader
         try
+            self:Connection:ForceOpen()
             if String.IsNullOrEmpty(cTable)
                 cTable := SELF:Name
             endif
@@ -202,6 +205,7 @@ class SqlDbCommand inherit SqlDbHandleObject implements IDisposable
     /// <seealso cref="SqlDbConnection.LastException"/>
     method ExecuteNonQuery(cTable := "" as string) as LOGIC
         try
+            self:Connection:ForceOpen()
             if String.IsNullOrEmpty(cTable)
                 cTable := SELF:Name
             endif

src/Runtime/XSharp.SQLRdd/Classes/Connection.prg

@@ -47,6 +47,9 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
     private _metadataCollections as List<string>
     private _databaseRestrictions as int
     private _tableRestrictions as int
+    // Must be kept in a field: a System.Timers.Timer with no other root gets garbage
+    // collected (it stops firing silently), which would disable the periodic stale-lock cleanup.
+    private _lockTimer as System.Timers.Timer
 
     #region Connection Only Properties
     /// <summary>Dictionary with properties defined by the Ado.Net provider</summary>
@@ -74,7 +77,17 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
     /// <summary>Provider for the Metadata, such as columnlist, maxrecords etc.</summary>
     property MetadataProvider   as ISqlMetadataProvider auto
     /// <summary>Last exception that occurred in the RDD</summary>
-    property LastException      as Exception auto get internal set
+    property LastException as Exception
+        get
+            return _lastException
+        end get
+        internal set
+            _lastException := value
+            if value != null
+                System.Diagnostics.Trace.WriteLine(String.Format("SqlDbConnection '{0}': {1}", self:Name, value:ToString()))
+            endif
+        end set
+    end property
     /// <summary>Connection State</summary>
     PROPERTY State              as ConnectionState get iif(self:DbConnection == null, ConnectionState.Closed, self:DbConnection:State)
 
@@ -264,6 +277,8 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         BufferSize         := DEFAULT_BUFFERSIZE
         DeletedColumn      := DEFAULT_DELETEDCOLUMN
         RecnoColumn        := DEFAULT_RECNOCOLUMN
+        LockRefreshInterval:= DEFAULT_LOCKREFRESHINTERVAL
+        LockStaleThreshold := DEFAULT_LOCKSTALETHRESHOLD
         cConnectionString  := self:AnalyzeConnectionString(cConnectionString)
         self:ConnectionString := cConnectionString
         DbConnection    := Provider:CreateConnection()
@@ -284,6 +299,9 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         SELF:_FillDataSourceProperties()
         SELF:_CreateLockTable()
         SELF:InitializeLockTimer()
+        // Sweep stale locks (e.g. left behind by a crashed/killed previous process) right away,
+        // instead of waiting for the first 120-second timer tick.
+        SELF:LockTimerElapsedEvent(null, null)
         // Todo: Check for # of open users and close the connection when no users are left and then throw an exception
         return
     end constructor
@@ -299,6 +317,11 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         if self:RDDs:Count > 0
             return false
         endif
+        if _lockTimer != null
+            _lockTimer:Enabled := false
+            _lockTimer:Dispose()
+            _lockTimer := null
+        endif
         // Logout the workstation from the Open Connections table
         if _command != null
             _command:Dispose()
@@ -368,7 +391,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
                 return false
             endif
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             self:DbTransaction := null
         end try
         return self:DbTransaction != null
@@ -388,7 +411,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
                 return false
             endif
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             self:DbTransaction := null
         end try
         return self:DbTransaction != null
@@ -447,7 +470,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
             _command:CommandText := cCommand
             result := _command:ExecuteScalar(cTable)
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             result := null
         end try
         return result
@@ -465,7 +488,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
              _command:CommandText := cCommand
             result := _command:ExecuteNonQuery(cTable)
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             result := false
         end try
         return result
@@ -483,7 +506,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
              _command:CommandText := cCommand
             result := _command:ExecuteReader(cTable)
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             result := null
         end try
         return result
@@ -501,7 +524,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
             _command:CommandText := cCommand
             result := _command:GetDataTable(cTable)
         catch e as Exception
-            _lastException := e
+            self:LastException := e
             result := null
         end try
         return result
@@ -583,7 +606,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
             next
             return oTd
         catch e as Exception
-            _lastException := e
+            self:LastException := e
         end try
         return null
     end method
@@ -613,7 +636,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
             self:Schema:Add(TableName, oTd)
             return oTd
         catch e as Exception
-            _lastException := e
+            self:LastException := e
         end try
         return null
     end method
@@ -630,6 +653,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         // DROP TABLE) causes DoesTableExist() to keep reporting a just-dropped table as
         // existing, which then skips re-creating it.
         try
+            self:ForceOpen()
             if !SELF:HasCollection(TABLECOLLECTION)
                 return false
             endif
@@ -650,7 +674,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
                 next
             endif
         catch e as Exception
-            _lastException := e
+            self:LastException := e
         end try
         return false
     end method
@@ -662,6 +686,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
     /// <param name="cDatabase">Name of the database to check</param>
     method DoesDatabaseExist(cDatabase as string) as logic
         try
+            self:ForceOpen()
             if !SELF:HasCollection(DATABASECOLLECTION)
                     return false
             endif
@@ -682,7 +707,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
                 next
             endif
         catch e as Exception
-            _lastException := e
+            self:LastException := e
         end try
         return false
     end method
@@ -693,6 +718,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
     /// <returns>List of table names that match the filter</returns>
     method GetTables(filter := "" as string) as List<string>
         try
+            self:ForceOpen()
             var result := List<string>{}
             if self:HasCollection(TABLECOLLECTION)
                 var dt := self:DbConnection:GetSchema(TABLECOLLECTION)
@@ -709,7 +735,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
             endif
             return result
         catch e as Exception
-            _lastException := e
+            self:LastException := e
         end try
         return null
     end method
@@ -719,6 +745,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
  /// </summary>
  /// <returns>List of metadata collections</returns>
     method GetMetaDataCollections() as List<string>
+        self:ForceOpen()
         var dt := self:DbConnection:GetSchema(DbMetaDataCollectionNames.MetaDataCollections)
         var result := List<string>{}
         foreach row as DataRow in dt:Rows
@@ -737,6 +764,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
 /// </summary>
 /// <returns>List of metadata collections</returns>
    method GetMetaDataCollection(cCollection as string) as DataTable
+       self:ForceOpen()
        var dt := self:DbConnection:GetSchema(cCollection)
        return dt
    end method
@@ -751,6 +779,39 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
 
     #endregion
 
+    // Interval at which a connection refreshes the timestamp on its own locks, and the age at
+    // which another connection's lock is considered abandoned and cleared. The threshold must
+    // stay comfortably above the refresh interval: with no margin, a lock could be judged
+    // abandoned just before its owner's own refresh tick runs, e.g. under GC/scheduler jitter,
+    // and a still-active user on another workstation (or another instance) would lose their lock.
+    // Both are overridable per connection, via ini or callback, same as PageSize/BufferSize -
+    // see SqlRDDEventReason.LockRefreshInterval/LockStaleThreshold and Metadata/Abstract.prg.
+    INTERNAL CONST DEFAULT_LOCKREFRESHINTERVAL := 120 AS INT
+    INTERNAL CONST DEFAULT_LOCKSTALETHRESHOLD := 600 AS INT
+
+    /// <summary>
+    /// The interval (in seconds) at which this connection refreshes the timestamp on its own
+    /// locks and sweeps locks older than <see cref="LockStaleThreshold"/> (also in seconds).
+    /// </summary>
+    property LockRefreshInterval as int
+        get
+            return _lockRefreshInterval
+        end get
+        set
+            _lockRefreshInterval := value
+            if _lockTimer != null
+                _lockTimer:Interval := value * 1000
+            endif
+        end set
+    end property
+    private _lockRefreshInterval as int
+
+    /// <summary>
+    /// The age (in seconds) after which another connection's lock is considered abandoned and
+    /// cleared. Must stay comfortably above <see cref="LockRefreshInterval"/> (see remarks above).
+    /// </summary>
+    property LockStaleThreshold as int auto
+
     INTERNAL CONST DEFAULT_ALLOWUPDATES := TRUE AS LOGIC
     INTERNAL CONST DEFAULT_COMPAREMEMO := TRUE AS LOGIC
     INTERNAL CONST DEFAULT_DELETEDCOLUMN := "" AS STRING
@@ -854,10 +915,10 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
     end method
 
     private method InitializeLockTimer() as void
-        var timer := System.Timers.Timer{120000} // 120 sec
-        timer:Elapsed += System.Timers.ElapsedEventHandler{ @@LockTimerElapsedEvent }
-        timer:AutoReset := true
-        timer:Enabled := true
+        _lockTimer := System.Timers.Timer{SELF:LockRefreshInterval * 1000}
+        _lockTimer:Elapsed += System.Timers.ElapsedEventHandler{ @@LockTimerElapsedEvent }
+        _lockTimer:AutoReset := true
+        _lockTimer:Enabled := true
     return
 
     method LockTimerElapsedEvent(sender as object, e as System.Timers.ElapsedEventArgs) as void
@@ -883,7 +944,7 @@ class SqlDbConnection inherit SqlDbHandleObject implements IDisposable
         endif
         cmdClear:CommandText := SELF:Provider:DeleteStatement:Replace(SqlDbProvider.TableNameMacro, LockTableName):Replace(SqlDbProvider.WhereMacro, "LockDateTime < " + parameterName1)
         cmdClear:Parameters := List<SqlDbParameter>{}
-        cmdClear:Parameters:Add(SqlDbParameter{parameterName1, DateTime.Now.AddSeconds(-120)})
+        cmdClear:Parameters:Add(SqlDbParameter{parameterName1, DateTime.Now.AddSeconds(-SELF:LockStaleThreshold)})
         cmdClear:ExecuteNonQuery()
     return
 

src/Runtime/XSharp.SQLRdd/Metadata/Abstract.prg

@@ -81,6 +81,8 @@ ABSTRACT CLASS SqlMetadataProviderAbstract IMPLEMENTS ISqlMetadataProvider
             _connection:AllowUpdates        := SELF:GetLogic(oPar,  SqlRDDEventReason.AllowUpdates, _connection:AllowUpdates)
             _connection:PageSize            := SELF:GetInt(oPar,    SqlRDDEventReason.PageSize, _connection:PageSize)
             _connection:BufferSize          := SELF:GetInt(oPar,    SqlRDDEventReason.BufferSize, _connection:BufferSize)
+            _connection:LockRefreshInterval := SELF:GetInt(oPar,    SqlRDDEventReason.LockRefreshInterval, _connection:LockRefreshInterval)
+            _connection:LockStaleThreshold  := SELF:GetInt(oPar,    SqlRDDEventReason.LockStaleThreshold, _connection:LockStaleThreshold)
             _connection:RecnoColumn         := SELF:GetString(oPar, SqlRDDEventReason.RecnoColumn, _connection:RecnoColumn)
             _connection:DeletedColumn       := SELF:GetString(oPar, SqlRDDEventReason.DeletedColumn, _connection:DeletedColumn)
             _connection:CompareMemo         := SELF:GetLogic(oPar,  SqlRDDEventReason.CompareMemo, _connection:CompareMemo)

src/Runtime/XSharp.SQLRdd/RDD/SQLDbOrder.prg

@@ -258,6 +258,15 @@ internal class SqlDbOrder inherit SqlDbObject
         if seekInfo:Value is string var strValue
             var strLen := strValue:Length
             if strLen < self:KeyLength
+                var cColumnCondition := SELF:BuildColumnAwareCondition(strValue, seekInfo:SoftSeek, cComp)
+                if cColumnCondition != null
+                    return cColumnCondition
+                endif
+                // Fallback for keys we can't map onto individual columns (functions in the key
+                // expression, unknown column metadata, ...): the original, always-correct
+                // condition against the fully concatenated SQLKey expression. SQL Server cannot
+                // use a normal index to seek into this - it has to compute the concatenation for
+                // every row - so prefer the column-aware path above whenever possible.
                 if (! seekInfo:SoftSeek)
                     cComp := " like "
                     strValue += "%"
@@ -278,5 +287,72 @@ internal class SqlDbOrder inherit SqlDbObject
         return cWhereClause
     end method
 
+    /// <summary>
+    /// Build a seek/scope condition against the individual columns that make up a plain
+    /// concatenated key (e.g. ALORT+NAMEUMLAUT), instead of against the fully concatenated
+    /// SQLKey expression.
+    /// </summary>
+    /// <remarks>
+    /// A condition like `[ALORT]+[NAMEUMLAUT] LIKE 'X%'` cannot use a normal index on
+    /// (ALORT, NAMEUMLAUT, ...) - SQL Server has to evaluate the concatenation for every row.
+    /// A value that covers one or more leading columns exactly can instead be expressed as
+    /// `[ALORT] = 'X' AND [NAMEUMLAUT] LIKE 'Y%'`, which is a normal composite-index seek.
+    /// Returns NULL when the key has functions in it, or column metadata/widths can't be
+    /// determined - callers must fall back to the concatenation-based condition in that case.
+    /// </remarks>
+    private method BuildColumnAwareCondition(strValue as string, lSoftSeek as logic, cRangeComp as string) as string
+        if self:HasFunctions
+            return null
+        endif
+        var columns := self:ColumnList
+        if columns == null .or. columns:Count == 0
+            return null
+        endif
+        var widths := List<int>{}
+        foreach var cCol in columns
+            var cName := cCol:Trim(<char>{'[',']','"','`'})
+            var oCol := self:RDD:TableColumns:FirstOrDefault({ c => String.Compare(c:Name, cName, true) == 0 })
+            if oCol == null
+                return null
+            endif
+            widths:Add((int) oCol:Length)
+        next
+
+        var sb := StringBuilder{}
+        var cRemaining := strValue
+        for var i := 0 upto columns:Count-1
+            if cRemaining:Length == 0
+                exit
+            endif
+            var nWidth   := widths[i]
+            var cColExpr := columns[i]
+            if sb:Length > 0
+                sb:Append(SqlDbProvider.AndClause)
+            endif
+            if cRemaining:Length >= nWidth
+                // The seek value fully covers this column - pin it down with equality and
+                // carry on with whatever's left over into the next column.
+                var cPart := cRemaining:Substring(0, nWidth)
+                sb:Append(cColExpr + " = " + Functions.XsValueToSqlValue(cPart))
+                cRemaining := cRemaining:Substring(nWidth)
+            else
+                // Partial match on this column - the same range/prefix logic the concatenation
+                // fallback uses, just scoped to this one column so the index can still be used.
+                if !lSoftSeek
+                    sb:Append(cColExpr + " like " + Functions.XsValueToSqlValue(cRemaining + "%"))
+                else
+                    var cSubstr := Provider:GetFunction("SUBSTR(%1%,%2%,%3%)")
+                    cSubstr := cSubstr:Replace("%1%", cColExpr):Replace("%2%","1"):Replace("%3%", cRemaining:Length:ToString())
+                    sb:Append(cSubstr + cRangeComp + Functions.XsValueToSqlValue(cRemaining))
+                endif
+                cRemaining := ""
+            endif
+        next
+        if sb:Length == 0
+            return null
+        endif
+        return sb:ToString()
+    end method
+
 end class
 end namespace // XSharp.RDD.SqlRDD