Updated template and job creation for scrapers and indexers

config_generation/db_to_xml.py

@@ -148,35 +148,265 @@ def convert_template_to_scraper(self, collection) -> None:
         scraper_config = self.update_config_xml()
         return scraper_config
 
-    def convert_template_to_plugin_indexer(self, scraper_editor) -> None:
+    def convert_template_to_job(self, collection, job_type) -> None:
         """
-        assuming this class has been instantiated with the scraper_template.xml
+        assuming this class has been instantiated with the job_template.xml
+        """
+        if job_type == "scrapers":
+            self.update_or_add_element_value("Collection", f"/scrapers/{collection.config_folder}/")
+        elif job_type == "indexer":
+            self.update_or_add_element_value("Collection", f"/SDE/{collection.config_folder}/")
+        job_config = self.update_config_xml()
+        return job_config
+
+    def convert_template_to_indexer(self, scraper_editor) -> None:
+        """
+        assuming this class has been instantiated with the final_config_template.xml
         """
 
         transfer_fields = [
-            "KeepHashFragmentInUrl",
+            "connector",
+            "description",
+            "identity",
+            "indexers",
+            "index",
+            "domain",
+            "treeRoot",
+            "Revision",
+            "visibility",
+            "ForceReindexation",
+            "WorkerCount",
+            "MaxWorkerPerHost",
+            "EnableNeuralIndexing",
+            "NeuralSearchSelectionQuery",
+            "MaxLevel",
+            "MaxToIndex",
+            "MaxToCrawl",
+            "MaxRedirection",
+            "CrawlMaxSize",
+            "CrawlTimeout",
+            "NormalizeUrls",
             "CorrectDomainCookies",
             "IgnoreSessionCookies",
-            "DownloadImages",
-            "DownloadMedia",
             "DownloadCss",
             "DownloadFtp",
             "DownloadFile",
             "IndexJs",
             "FollowJs",
             "CrawlFlash",
-            "NormalizeSecureSchemesWhenTestingVisited",
+            "IndexEmptyPages",
+            "CrawlWebsphereSeedlist",
+            "KeepHashFragmentInUrl",
             "RetryCount",
             "RetryPause",
+            "HttpCodesToRetry",
+            "UseIfModifiedSince",
+            "UseIfNoneMatch",
+            "AcceptWeakETag",
+            "ForcedEncoding",
+            "UseCompression",
+            "UseUnsafeHeaderParsing",
+            "NormalizeSecureSchemesWhenTestingVisited",
+            "ExactDeduplication",
+            "NearDeduplication",
+            "CrawlPauseDelay",
+            "CrawlPauseCount",
+            "UseRuntimeAutoRedirect",
+            "RememberDnsFailure",
+            "RememberConnectFailure",
+            "RememberTrustFailure",
+            "RememberProxyNameResolutionFailure",
+            "UseRobotsNoFollow",
+            "UseRobotsTxt",
+            "RobotsTxtCaseSensitive",
+            "LoadRobotsTxtSitemapUrls",
+            "CheckSitemapUrlLastmodInRealtimeMode",
+            "AddRobotsTxtAllowUrlsToSeedList",
+            "UseCanonicalLinks",
+            "UseRelNoFollow",
+            "DownloadSelectionQuery",
+            "FollowSelectionQuery",
+            "IndexSelectionQuery",
+            "LoadDefaultTags",
+            "LoadDefaultJsTransforms",
+            "DisplayLongProperties",
+            "LongPropertyLimit",
+            "UsePerformanceMetrics",
+            "LogPerformanceMetricsPeriodically",
+            "LogPerformanceMetricsPeriod",
+            "PasswordRepository",
+            "StoreDocumentCache",
+            "AuditEnabled",
+            "SaveDeniedDocs",
+            "SavePropertiesToRegistry",
+            "CollectionStateNative",
+            "XPathNavigatorNative",
+            "StatusMaxOk",
+            "DelApiSecret",
+            "RealTimeIncrementalState",
+            "RealTimeInfoOnError",
+            "ConversionProxies",
+            "ConversionPlan",
             "AddBaseHref",
             "AddMetaContentType",
-            "NormalizeUrls",
+            "Throttle",
+            "DocumentClass",
+            "ConnectorLanguage",
+            "ClearHttpRequestCanonicalizeAsFilePath",
+            "IndexZipContent",
+            "IndexPdfAttachments",
+            "IndexOleAttachments",
+            "IndexMsgContent",
+            "IndexMsgAttachments",
+            "IndexOftContent",
+            "IndexOftAttachments",
+            "IndexEmlContent",
+            "IndexEmlAttachments",
+            "IndexPstContent",
+            "IndexOstContent",
+            "IndexPstMsg",
+            "IndexPstMsgAttachments",
+            "IndexPstContact",
+            "IndexPstCalendar",
+            "IndexPstNote",
+            "IndexPstTask",
+            "IndexPstDocument",
+            "PstUseSafeId",
+            "IndexArchivesExtensions",
+            "ArchiveItemsUseArchiveVersion",
+            "UseShortAttachmentId",
+            "UseExtendedExtensionGuesser",
+            "AlwaysScanContainerFiles",
+            "XmpExtensions",
+            "MediaExtensions",
+            "ExiftoolExtensions",
+            "EarlySelectionQuery",
+            "SelectionQuery",
+            "AttachmentSelectionQuery",
+            "ArchiveItemSelectionQuery",
+            "EngineConnectionWait",
+            "FetchCollectionDataDirectlyFromEngine",
+            "CalculateGraphBoost",
+            "GraphBoostColumn",
+            "GraphBoostEMColumn",
+            "GraphBoostIterations",
+            "GraphBoostPower",
+            "GraphBoostAdd",
+            "UseFieldPermissions",
+            "ShardIndexes",
+            "ShardingStrategy",
+            "ShardSelections",
+            "CurationType",
+            "CurationIdPattern",
+            "RunIndexMiningInIndexer",
+            "Namespace",
         ]
 
         double_transfer_fields = [
+            ("UrlAccess", "UseDefaultCredentials"),
+            ("UrlAccess", "UseDefaultNetworkCredentials"),
+            ("UrlAccess", "User"),
+            ("UrlAccess", "Password"),
+            ("UrlAccess", "Domain"),
+            ("UrlAccess", "UseRfc1945"),
+            ("UrlAccess", "Timeout"),
+            ("UrlAccess", "ChangeConnectionGroupNameOnTimeout"),
+            ("UrlAccess", "AllowAuthenticatedConnectionSharing"),
+            ("UrlAccess", "PreAuthenticate"),
+            ("UrlAccess", "HttpVersion"),
+            ("UrlAccess", "KeepAlive"),
+            ("UrlAccess", "SecurityProtocol"),
+            ("UrlAccess", "UserAgent"),
+            ("UrlAccess", "ClientCertificateFile"),
+            ("UrlAccess", "ClientCertificatePassword"),
+            ("UrlAccess", "ClientCertificateStorage"),
             ("UrlAccess", "AllowXPathCookies"),
-            ("UrlAccess", "UseBrowserForWebRequests"),
             ("UrlAccess", "UseHttpClientForWebRequests"),
+            ("UrlAccess", "ThrottleManagerCode"),
+            ("UrlAccess", "UseBrowserForWebRequests"),
+            ("UrlAccess", "BrowserForWebRequestsReadinessThreshold"),
+            ("UrlAccess", "BrowserForWebRequestsInitialDelay"),
+            ("UrlAccess", "BrowserForWebRequestsMaxTotalDelay"),
+            ("UrlAccess", "BrowserForWebRequestsMaxResourcesDelay"),
+            ("UrlAccess", "BrowserForWebRequestsLogLevel"),
+            ("UrlAccess", "BrowserForWebRequestsViewportWidth"),
+            ("UrlAccess", "BrowserForWebRequestsViewportHeight"),
+            ("UrlAccess", "BrowserForWebRequestsAdditionalJavascript"),
+            ("UrlAccess", "WebConnectionPluginName"),
+            ("UrlAccess", "PostLoginUrl"),
+            ("UrlAccess", "PostLoginData"),
+            ("UrlAccess", "GetBeforePostLogin"),
+            ("UrlAccess", "PostLoginAutoRedirect"),
+            ("UrlAccess", "ReLoginCount"),
+            ("UrlAccess", "ReLoginDelay"),
+            ("UrlAccess", "DetectHtmlLoginPattern"),
+            ("UrlAccess", "FtpUser"),
+            ("UrlAccess", "FtpPassword"),
+            ("UrlAccess", "FtpDomain"),
+            ("UrlAccess", "FtpUseBinary"),
+            ("UrlAccess", "FtpUsePassive"),
+            ("UrlAccess", "FtpReadWriteTimeout"),
+            ("UrlAccess", "FtpTimeout"),
+            ("UrlAccess", "FtpEnableSsl"),
+            ("UrlAccess", "FileUser"),
+            ("UrlAccess", "FilePassword"),
+            ("UrlAccess", "FileDomain"),
+            ("UrlAccess", "FileTimeout"),
+            ("UrlAccess", "ProxyAutoDetect"),
+            ("UrlAccess", "ProxyAddress"),
+            ("UrlAccess", "ProxyBypassOnLocal"),
+            ("UrlAccess", "ProxyServer"),
+            ("UrlAccess", "ProxyPort"),
+            ("UrlAccess", "ProxyUseDefaultCredentials"),
+            ("UrlAccess", "ProxyUseDefaultNetworkCredentials"),
+            ("UrlAccess", "ProxyUser"),
+            ("UrlAccess", "ProxyPassword"),
+            ("UrlAccess", "ProxyDomain"),
+            ("IndexerClient", "Simulate"),
+            ("IndexerClient", "SimulateGetCollectionState"),
+            ("IndexerClient", "QueueMaxCount"),
+            ("IndexerClient", "SendingThreadFactor"),
+            ("IndexerClient", "DirectFileAccess"),
+            ("IndexerClient", "UseCompression"),
+            ("IndexerClient", "SessionIsFinishedWait"),
+            ("IndexerClient", "SendTimeout"),
+            ("IndexerClient", "RetryConnectCount"),
+            ("IndexerClient", "RetryConnectDelay"),
+            ("IndexerClient", "RetryTimeout"),
+            ("IndexerClient", "RetrySleep"),
+            ("IndexerClient", "DeactivationTimeout"),
+            ("Indexation", "SimulateLemma"),
+            ("Indexation", "SimulateEngine"),
+            ("Indexation", "SimulateCache"),
+            ("Indexation", "SimulateLemmaMin"),
+            ("Indexation", "SimulateLemmaMax"),
+            ("Indexation", "CollectionStateParallelRowFetch"),
+            ("Indexation", "EngineMetaEnabled"),
+            ("Indexation", "ThumbnailHeight"),
+            ("Indexation", "ThumbnailWidth"),
+            ("Indexation", "ThumbnailSmallTimeout"),
+            ("Indexation", "ThumbnailMediumTimeout"),
+            ("Indexation", "ThumbnailLargeTimeout"),
+            ("Indexation", "SynchThumbnailGen"),
+            ("Indexation", "StoreInCollectionCache"),
+            ("Indexation", "GetFilePropertiesFromConverter"),
+            ("PdfGen", "ConverterType"),
+            ("PdfGen", "TimeoutSmall"),
+            ("PdfGen", "TimeoutMedium"),
+            ("PdfGen", "TimeoutLarge"),
+        ]
+
+        triple_transfer_fields = [
+            ("UrlAccess", "BrowserLogin", "Activate"),
+            ("UrlAccess", "BrowserLogin", "RemoteDebuggingPort"),
+            ("UrlAccess", "BrowserLogin", "BrowserLogLevel"),
+            ("UrlAccess", "BrowserLogin", "ShowDevTools"),
+            ("UrlAccess", "BrowserLogin", "SuccessCondition"),
+            ("UrlAccess", "BrowserLogin", "CookieFilter"),
+            ("UrlAccess", "AmazonS3", "AccessKey"),
+            ("UrlAccess", "AmazonS3", "SecretKey"),
+            ("UrlAccess", "AmazonS3", "RegionEndpoint"),
+            ("UrlAccess", "AmazonS3", "ServiceURL"),
         ]
 
         for field in transfer_fields:
@@ -187,18 +417,15 @@ def convert_template_to_plugin_indexer(self, scraper_editor) -> None:
                 f"{parent}/{child}", scraper_editor.get_tag_value(f"{parent}/{child}", strict=True)
             )
 
+        for grandparent, parent, child in triple_transfer_fields:
+            self.update_or_add_element_value(
+                f"{grandparent}/{parent}/{child}",
+                scraper_editor.get_tag_value(f"{grandparent}/{parent}/{child}", strict=True),
+            )
+
         scraper_config = self.update_config_xml()
         return scraper_config
 
-    def convert_template_to_indexer(self, collection) -> None:
-        """
-        assuming this class has been instantiated with the indexer_template.xml
-        """
-        self.update_or_add_element_value("Collection", f"/SDE/{collection.config_folder}/")
-        indexer_config = self.update_config_xml()
-
-        return indexer_config
-
     def _mapping_exists(self, new_mapping: ET.Element):
         """
         Check if the mapping with given parameters already exists in the XML tree

config_generation/generate_jobs.py

@@ -39,7 +39,7 @@ def _create_job_name(self, collection_name):
         if source == "SDE":
             return f"collection.indexer.{collection_name}.xml"
         else:
-            return f"collection.indexer.{source}.{collection_name}.xml"
+            return f"collection.{source}.{collection_name}.xml"
 
     def _create_joblist_name(self, index):
         """