Minor fixes

Algorithm.CSharp/BasicTemplateFuturesWithExtendedMarketDailyAlgorithm.cs

@@ -14,14 +14,7 @@
  *
 */
 
-using System;
 using System.Collections.Generic;
-using System.Linq;
-using QuantConnect.Data;
-using QuantConnect.Interfaces;
-using QuantConnect.Orders;
-using QuantConnect.Securities;
-using QuantConnect.Securities.Future;
 
 namespace QuantConnect.Algorithm.CSharp
 {
@@ -43,40 +36,40 @@ public class BasicTemplateFuturesWithExtendedMarketDailyAlgorithm : BasicTemplat
         /// <summary>
         /// Data Points count of all timeslices of algorithm
         /// </summary>
-        public override long DataPoints => 14895;
+        public override long DataPoints => 14883;
 
         /// <summary>
         /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
         /// </summary>
         public override Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
         {
-            {"Total Orders", "36"},
+            {"Total Orders", "32"},
             {"Average Win", "0.33%"},
-            {"Average Loss", "-0.03%"},
-            {"Compounding Annual Return", "0.102%"},
+            {"Average Loss", "-0.04%"},
+            {"Compounding Annual Return", "0.110%"},
             {"Drawdown", "0.300%"},
-            {"Expectancy", "0.171"},
+            {"Expectancy", "0.184"},
             {"Start Equity", "1000000"},
-            {"End Equity", "1001024.4"},
-            {"Net Profit", "0.102%"},
-            {"Sharpe Ratio", "-1.702"},
-            {"Sortino Ratio", "-0.836"},
-            {"Probabilistic Sharpe Ratio", "14.653%"},
-            {"Loss Rate", "89%"},
-            {"Win Rate", "11%"},
-            {"Profit-Loss Ratio", "9.54"},
+            {"End Equity", "1001108"},
+            {"Net Profit", "0.111%"},
+            {"Sharpe Ratio", "-1.688"},
+            {"Sortino Ratio", "-0.772"},
+            {"Probabilistic Sharpe Ratio", "14.944%"},
+            {"Loss Rate", "88%"},
+            {"Win Rate", "12%"},
+            {"Profit-Loss Ratio", "8.47"},
             {"Alpha", "-0.007"},
             {"Beta", "0.002"},
             {"Annual Standard Deviation", "0.004"},
             {"Annual Variance", "0"},
             {"Information Ratio", "-1.353"},
             {"Tracking Error", "0.089"},
-            {"Treynor Ratio", "-4.126"},
-            {"Total Fees", "$80.60"},
+            {"Treynor Ratio", "-4.099"},
+            {"Total Fees", "$72.00"},
             {"Estimated Strategy Capacity", "$0"},
             {"Lowest Capacity Asset", "ES VRJST036ZY0X"},
-            {"Portfolio Turnover", "0.97%"},
-            {"OrderListHash", "52c852d720692fab1e12212b2aba03d4"}
+            {"Portfolio Turnover", "0.87%"},
+            {"OrderListHash", "ef59fd5e4a7ae483a60d25736cf5d2d8"}
         };
     }
 }

Algorithm.CSharp/FillForwardTestAlgorithm.cs → Algorithm.CSharp/StrictEndTimeLowerResolutionFillForwardRegressionAlgorithm.cs

@@ -30,7 +30,7 @@ namespace QuantConnect.Algorithm.CSharp
     /// 1. Test that the on-consolidated event is not called for fill forwarded data in identity and higher period consolidators
     /// 2. Test that the intra-day fill-forwarded data is not fed to indicators
     /// </summary>
-    public class FillForwardTestAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    public class StrictEndTimeLowerResolutionFillForwardRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
     {
         private Equity _aapl;
 

Engine/DataFeeds/Enumerators/FillForwardEnumerator.cs

@@ -381,7 +381,11 @@ protected virtual bool RequiresFillForwardData(TimeSpan fillForwardResolution, B
                         if (_useStrictEndTime)
                         {
                             // TODO: what about extended market hours
-                            expectedPeriod = Exchange.Hours.RegularMarketDuration;
+                            expectedPeriod = Exchange.Hours.GetMarketHours(potentialBarEndTimeInExchangeTZ).MarketDuration;
+                            if (expectedPeriod == TimeSpan.Zero)
+                            {
+                                expectedPeriod = Exchange.Hours.GetMarketHours(nextFillForwardBarStartTime).MarketDuration;
+                            }
                         }
                         fillForward.Time = (potentialBarEndTime - expectedPeriod).ConvertFromUtc(Exchange.TimeZone);
                         fillForward.EndTime = potentialBarEndTimeInExchangeTZ;

Engine/DataFeeds/SubscriptionData.cs

@@ -102,7 +102,8 @@ public static SubscriptionData Create(bool dailyStrictEndTimeEnabled, Subscripti
                 // (which is correct, since the last daily bar belongs to the previous date).
                 // If this is a fill-forwarded complete daily bar (ending at market close),
                 // the daily calendar will have the same time/end time so the bar times will not be adjusted.
-                var calendar = LeanData.GetDailyCalendar(data.Time, exchangeHours, configuration.ExtendedMarketHours);
+                // TODO: What about extended market hours? How to handle non-adjacent market hour segments in a day? Same in FillForwardEnumerator
+                var calendar = LeanData.GetDailyCalendar(data.Time, exchangeHours, false);
                 data.Time = calendar.Start;
                 data.EndTime = calendar.End;
             }

Tests/Engine/DataFeeds/Enumerators/FillForwardEnumeratorTests.cs

@@ -126,7 +126,9 @@ public void GetReferenceDateIntervals_RoundDown(bool strictEndTimes)
 
             Assert.IsTrue(fillForwardEnumerator.MoveNext());
             // Time should advance!
-            Assert.AreEqual(new DateTime(2017, 7, 22, 17, 1, 0), fillForwardEnumerator.Current.Time);
+            // Time should be 10:01am to 5:01pm, on Sundays the market opens at 5pm, so the market duration is 7 hours
+            var expectedTime = strictEndTimes ? new DateTime(2017, 7, 23, 10, 1, 0) : new DateTime(2017, 7, 22, 17, 1, 0);
+            Assert.AreEqual(expectedTime, fillForwardEnumerator.Current.Time);
             Assert.AreEqual(new DateTime(2017, 7, 23, 17, 1, 0), fillForwardEnumerator.Current.EndTime);
             Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
             Assert.AreEqual(0, (fillForwardEnumerator.Current as TradeBar).Volume);
@@ -774,6 +776,80 @@ public void FillsForwardDailyMissingDays(bool strictEndTimes)
             fillForwardEnumerator.Dispose();
         }
 
+        [Test]
+        public void FillForwardsDailyMissingDaysRespectingEarlyClose()
+        {
+            var symbol = Symbols.SPY;
+            var dataResolution = Time.OneDay;
+            var commonMarketDuration = new TimeSpan(6, 30, 0);
+            var startTimeOfDay = new TimeSpan(9, 30, 0);
+            var reference = new DateTime(2015, 11, 25).Add(startTimeOfDay);
+
+            var data = new BaseData[]
+            {
+                // wed 11/25
+                new TradeBar {Value = 0, Time = reference, Period = commonMarketDuration, Volume = 100},
+                // tue 12/1
+                new TradeBar {Value = 1, Time = reference.AddDays(6), Period = commonMarketDuration, Volume = 200},
+            }.ToList();
+            var enumerator = data.GetEnumerator();
+
+            var exchange = new SecurityExchange(MarketHoursDatabase.FromDataFolder().GetExchangeHours(symbol.ID.Market, symbol, symbol.SecurityType));
+            var isExtendedMarketHours = false;
+            using var fillForwardEnumerator = new FillForwardEnumerator(enumerator, exchange, Ref.Create(TimeSpan.FromDays(1)), isExtendedMarketHours,
+                data[^1].EndTime.Date.AddDays(1), dataResolution, exchange.TimeZone, dailyStrictEndTimeEnabled: true);
+
+            var dataReferenceTime = reference;
+            var dataReferenceEndTime = reference.Add(commonMarketDuration);
+
+            // wed 11/25
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.AreEqual(dataReferenceTime, fillForwardEnumerator.Current.Time);
+            Assert.AreEqual(dataReferenceEndTime, fillForwardEnumerator.Current.EndTime);
+            Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
+            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(commonMarketDuration, ((TradeBar)fillForwardEnumerator.Current).Period);
+            Assert.AreEqual(100, ((TradeBar)fillForwardEnumerator.Current).Volume);
+
+            // thu 11/26 (no data, holiday)
+
+            // fri 11/27 (early close: 1pm)
+            dataReferenceTime = dataReferenceTime.AddDays(2);
+            dataReferenceEndTime = dataReferenceEndTime.AddDays(2);
+            var earlyClose = dataReferenceEndTime.Date.Add(TimeSpan.FromHours(13));
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.AreEqual(dataReferenceTime, fillForwardEnumerator.Current.Time);
+            Assert.AreEqual(earlyClose, fillForwardEnumerator.Current.EndTime);
+            Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
+            Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(earlyClose - dataReferenceTime, ((TradeBar)fillForwardEnumerator.Current).Period);
+            Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
+
+            // mon 11/30
+            dataReferenceTime = dataReferenceTime.AddDays(3);
+            dataReferenceEndTime = dataReferenceEndTime.AddDays(3);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.AreEqual(dataReferenceTime, fillForwardEnumerator.Current.Time);
+            Assert.AreEqual(dataReferenceEndTime, fillForwardEnumerator.Current.EndTime);
+            Assert.AreEqual(0, fillForwardEnumerator.Current.Value);
+            Assert.IsTrue(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(commonMarketDuration, ((TradeBar)fillForwardEnumerator.Current).Period);
+            Assert.AreEqual(0, ((TradeBar)fillForwardEnumerator.Current).Volume);
+
+            // tue 12/1
+            dataReferenceTime = dataReferenceTime.AddDays(1);
+            dataReferenceEndTime = dataReferenceEndTime.AddDays(1);
+            Assert.IsTrue(fillForwardEnumerator.MoveNext());
+            Assert.AreEqual(dataReferenceTime, fillForwardEnumerator.Current.Time);
+            Assert.AreEqual(dataReferenceEndTime, fillForwardEnumerator.Current.EndTime);
+            Assert.AreEqual(1, fillForwardEnumerator.Current.Value);
+            Assert.IsFalse(fillForwardEnumerator.Current.IsFillForward);
+            Assert.AreEqual(commonMarketDuration, ((TradeBar)fillForwardEnumerator.Current).Period);
+            Assert.AreEqual(200, ((TradeBar)fillForwardEnumerator.Current).Volume);
+
+            Assert.IsFalse(fillForwardEnumerator.MoveNext());
+        }
+
         [Test]
         public void FillsForwardHoursAtEndOfDayByHalfHour()
         {
@@ -1266,7 +1342,7 @@ public void FillsForwardBarsForDifferentResolutions(Resolution resolution, Resol
 #pragma warning disable CS0162 // Unreachable code detected; used to store expected data
                 QuantConnect.Compression.ZipCreateAppendData(
                     "../../TestData/FillForwardBars.zip", expectedDataFile, FillForwardTestAlgorithm.Result.Value, overrideEntry: true);
-#pragma warning restore CS0162 
+#pragma warning restore CS0162
             }
             QuantConnect.Compression.Unzip("TestData/FillForwardBars.zip", "./", overwrite: true);
             var expected = File.ReadAllLines(expectedDataFile);