Skip to content

Commit 57ec0ef

Browse files
AlexCatarinoclaude
AlexCatarino
and
claude
authored
Fix: don't downgrade map/factor-file provider based only on USA equity data (#653)
* fix: don't downgrade map/factor-file provider based only on USA equity data `_handle_data_providers` downgraded the global `map-file-provider` (and `factor-file-provider`) from the Zip provider to the Disk provider based solely on the state of `equity/usa/map_files`. The provider is a single global engine setting and the Disk provider silently ignores `map_files_*.zip` archives, so a futures-only data folder (whose map files ship only inside the zip) had its provider swapped out and continuous futures never mapped (`Mapped: None`) with no error raised. Gate the downgrade on every market's auxiliary folder instead of just `equity/usa`: only fall back to the Disk provider when there is no recent zip to lose for any market. Adds regression tests covering futures-only data, loose-csv-only data, and stale-zip data. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * refactor: use datetime.min sentinel to drop None checks Initialize newest_zip_date to datetime.min so the "no zip found" case naturally falls through the >7-days-old downgrade check, removing both explicit None checks. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * refactor: simplify provider gate, short-circuit on first recent zip, add debug logging - Glob the auxiliary zips directly (*/*/<dir>/*.zip) instead of walking dirs/files. - Return as soon as any market has a zip within the freshness window instead of scanning all markets for the newest date (equivalent decision, fewer iterations). - Hoist datetime.now() out of the loop. - Add debug logs for both outcomes (which provider is used and why). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 5d12659 commit 57ec0ef

2 files changed

Lines changed: 100 additions & 13 deletions

File tree

‎lean/components/docker/lean_runner.py‎

Lines changed: 35 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -463,12 +463,14 @@ def _handle_data_providers(self, lean_config: Dict[str, Any], data_dir: Path):
463463
"map-file-provider",
464464
"QuantConnect.Data.Auxiliary.LocalZipMapFileProvider",
465465
"QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider",
466-
data_dir / "equity" / "usa" / "map_files")
466+
data_dir,
467+
"map_files")
467468
self._force_disk_provider_if_necessary(lean_config,
468469
"factor-file-provider",
469470
"QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider",
470471
"QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider",
471-
data_dir / "equity" / "usa" / "factor_files")
472+
data_dir,
473+
"factor_files")
472474

473475
def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None:
474476
"""Sets up Docker run options specific to Python projects.
@@ -838,30 +840,50 @@ def _force_disk_provider_if_necessary(self,
838840
config_key: str,
839841
zip_provider: str,
840842
disk_provider: str,
841-
zip_dir: Path) -> None:
843+
data_dir: Path,
844+
auxiliary_dir_name: str) -> None:
842845
"""Updates the Lean config to use the disk provider instead of the zip one if there are no zips to use.
843846
847+
The map-file/factor-file provider is a single global engine setting that applies to every market.
848+
The zip providers read per-market '<securityType>/<market>/<auxiliary_dir_name>/<name>_yyyyMMdd.zip'
849+
archives, while the disk providers can only read loose '.csv' files and silently ignore those zip
850+
archives. We must therefore only downgrade to the disk provider when there is no recent zip to lose
851+
for *any* market, not just 'equity/usa'. Otherwise a futures-only data folder (whose map files ship
852+
only inside the zip) would have its zip provider swapped out and silently stop resolving, e.g.
853+
continuous futures would never map (Mapped: None) with no error raised.
854+
844855
:param lean_config: the Lean config to update
845856
:param config_key: the key of the configuration property
846857
:param zip_provider: the fully classified name of the zip provider for this property
847858
:param disk_provider: the fully classified name of the disk provider for this property
848-
:param zip_dir: the directory where the zip provider looks for zip files
859+
:param data_dir: the root data directory
860+
:param auxiliary_dir_name: the auxiliary subdirectory the zip provider reads ("map_files"/"factor_files")
849861
"""
850862
from re import sub
851863
from datetime import datetime
852864

853865
if lean_config.get(config_key, None) != zip_provider:
854866
return
855867

856-
if not zip_dir.exists():
857-
lean_config[config_key] = disk_provider
858-
return
859-
860-
zip_names = sorted([f.name for f in zip_dir.iterdir() if f.name.endswith(".zip")], reverse=True)
861-
zip_names = [sub(r"[^\d]", "", name) for name in zip_names]
862-
863-
if len(zip_names) == 0 or (datetime.now() - datetime.strptime(zip_names[0], "%Y%m%d")).days > 7:
864-
lean_config[config_key] = disk_provider
868+
# Keep the zip provider as long as any market has a recent map/factor file zip: the disk
869+
# provider only reads loose csv, so downgrading would silently drop zip-shipped files (e.g.
870+
# futures map files). We only need to know a recent zip exists, so we stop at the first one.
871+
now = datetime.now()
872+
for zip_file in data_dir.glob(f"*/*/{auxiliary_dir_name}/*.zip"):
873+
try:
874+
zip_date = datetime.strptime(sub(r"[^\d]", "", zip_file.name), "%Y%m%d")
875+
except ValueError:
876+
continue
877+
if (now - zip_date).days <= 7:
878+
self._logger.debug(
879+
f"LeanRunner._force_disk_provider_if_necessary(): found recent '{auxiliary_dir_name}' zip "
880+
f"'{zip_file.name}', keeping '{zip_provider}' for '{config_key}'")
881+
return
882+
883+
self._logger.debug(
884+
f"LeanRunner._force_disk_provider_if_necessary(): no '{auxiliary_dir_name}' zip newer than 7 days, "
885+
f"using '{disk_provider}' for '{config_key}'")
886+
lean_config[config_key] = disk_provider
865887

866888
def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file,
867889
set_up_common_csharp_options_called, release, image: DockerImage) -> None:

‎tests/components/docker/test_lean_runner.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
# See the License for the specific language governing permissions and
1212
# limitations under the License.
1313

14+
from datetime import datetime
1415
from pathlib import Path
1516
from unittest import mock
1617

@@ -91,6 +92,70 @@ def create_lean_runner(docker_manager: mock.Mock) -> LeanRunner:
9192
xml_manager)
9293

9394

95+
def test_handle_data_providers_keeps_zip_providers_for_futures_only_data() -> None:
96+
# Regression: a futures-only data folder has fresh map/factor file zips under future/cme but no
97+
# equity/usa data. The global zip providers must be kept; downgrading to the disk providers would
98+
# silently break futures map-file resolution (continuous futures would never map, Mapped: None).
99+
lean_runner = create_lean_runner(mock.Mock())
100+
101+
data_dir = Path.cwd() / "data"
102+
fresh = datetime.now().strftime("%Y%m%d")
103+
for auxiliary_dir_name in ["map_files", "factor_files"]:
104+
directory = data_dir / "future" / "cme" / auxiliary_dir_name
105+
directory.mkdir(parents=True, exist_ok=True)
106+
(directory / f"{auxiliary_dir_name}_{fresh}.zip").touch()
107+
108+
lean_config = {
109+
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
110+
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider",
111+
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider",
112+
}
113+
lean_runner._handle_data_providers(lean_config, data_dir)
114+
115+
assert lean_config["map-file-provider"] == "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider"
116+
assert lean_config["factor-file-provider"] == "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider"
117+
118+
119+
def test_handle_data_providers_downgrades_to_disk_providers_without_any_zip() -> None:
120+
# When the data folder only has loose csv auxiliary files (e.g. the free sample data) and no zips
121+
# for any market, fall back to the disk providers which read those loose files.
122+
lean_runner = create_lean_runner(mock.Mock())
123+
124+
data_dir = Path.cwd() / "data"
125+
directory = data_dir / "equity" / "usa" / "map_files"
126+
directory.mkdir(parents=True, exist_ok=True)
127+
(directory / "spy.csv").touch()
128+
129+
lean_config = {
130+
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
131+
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider",
132+
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider",
133+
}
134+
lean_runner._handle_data_providers(lean_config, data_dir)
135+
136+
assert lean_config["map-file-provider"] == "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider"
137+
assert lean_config["factor-file-provider"] == "QuantConnect.Data.Auxiliary.LocalDiskFactorFileProvider"
138+
139+
140+
def test_handle_data_providers_downgrades_to_disk_providers_when_zips_are_stale() -> None:
141+
# If the newest zip for every market is older than the freshness window, fall back to disk.
142+
lean_runner = create_lean_runner(mock.Mock())
143+
144+
data_dir = Path.cwd() / "data"
145+
directory = data_dir / "future" / "cme" / "map_files"
146+
directory.mkdir(parents=True, exist_ok=True)
147+
(directory / "map_files_20200101.zip").touch()
148+
149+
lean_config = {
150+
"data-provider": "QuantConnect.Lean.Engine.DataFeeds.DefaultDataProvider",
151+
"map-file-provider": "QuantConnect.Data.Auxiliary.LocalZipMapFileProvider",
152+
"factor-file-provider": "QuantConnect.Data.Auxiliary.LocalZipFactorFileProvider",
153+
}
154+
lean_runner._handle_data_providers(lean_config, data_dir)
155+
156+
assert lean_config["map-file-provider"] == "QuantConnect.Data.Auxiliary.LocalDiskMapFileProvider"
157+
158+
94159
@pytest.mark.parametrize("release", [False, True])
95160
def test_run_lean_compiles_csharp_project_in_correct_configuration(release: bool) -> None:
96161
create_fake_lean_cli_directory()

0 commit comments

Comments
 (0)