release...

modules/app.xql

@@ -591,7 +591,7 @@ declare function app:bulk-form-html($identifiers as xs:string*, $catalogs as xs:
     (
     <div>
         <h1>Bulk form for fast and efficient queries !</h1>
-        <form method="post" action="bulk.html"> (: force action to avoid param duplication on post :)
+        <form method="post" action="bulk.html"> <!-- force action to avoid param duplication on post -->
             <div class="d-flex p-2">
             <div class="input-group">
                 <input type="text" class="form-control" placeholder="Enter your science identifiers or coordinates. Use semicolon as separator, e.g : 0 0; 4.321 6.543; HD123; HD234 " aria-label="Science identifiers (semicolon separator)" id="identifiers" name="identifiers" value="{$identifiers}"/>
@@ -637,7 +637,7 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
     let $identifiers-map := $bulk-search-map?identifiers-map
 
     (: Rebuild the table (and votable) with a summary of what we have in the catalogs :)
-	let $log := util:log("info", "merge main table to show ranked results... ")
+	let $log := util:log("info", "merge main table to show ranked results ("|| count($identifiers-map?*)||"))... ")
 
     let $detail_cols := for $e in (array:flatten($app:conf?extended-cols)) return lower-case($e)
 
@@ -648,22 +648,17 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
     let $trs :=  for $identifier in map:keys($identifiers-map) order by $identifier
         let $science := map:get($identifiers-map, $identifier)
         return
-            for $cat in $catalogs-to-query
-                let $ftaos := $bulk-search-map($cat)?ranking?ftaos
-                let $scores := $bulk-search-map($cat)?ranking?scores
-                let $science-idx := $bulk-search-map($cat)?ranking?sciences-idx?($identifier)
-                let $inputs := $bulk-search-map($cat)?ranking?inputs
-            (:
-                        { count( $science-idx ) } configurations :
-                        <table class="table table-bordered table-light table-hover">
-                            <tr><th>FT</th><th>AO</th><th>SCORE</th></tr>9.03 16.381952728829486 4.91 16.883200012527162 31.668749771423702
-                            { for $idx in $science-idx return <tr><td>{$ftaos?*[$idx]?*[1]}</td><td>{$ftaos?*[$idx]?*[2]}</td><td>{$scores?*[$idx]}</td></tr> }
-                        </table>
-                :)
-                let $ordered-science-idx := for $idx in $science-idx order by $scores?*[$idx] descending return $idx
+            for $cat-name in map:keys($bulk-search-map?catalogs)
+                let $cat := $bulk-search-map?catalogs($cat-name)
+                let $ftaos := $cat?ranking?ftaos
+                let $scores := $cat?ranking?scores
+                let $science-idx := $cat?ranking?sciences-idx?($identifier)
+                let $inputs := $cat?ranking?inputs
+                let $ordered-science-idx := for $idx in $science-idx let $score:=$scores?*[$idx] where $score > 0 order by $score descending return $idx
+
             return
                 for $idx at $pos in $ordered-science-idx
-                    where number($scores?*[$idx]) > 0  and $pos <= $max?rank
+                    where $scores?*[$idx] > 0  and $pos <= $max?rank
                     let $ftao := $ftaos?*[$idx]?*
                     return
                         <tr>
@@ -673,17 +668,18 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
                             <td>{$scores?*[$idx]}</td>
                             <td>{$pos}</td>
                             { let $tds := array:flatten($inputs?*[$idx]) return for $c at $pos in $ranking-input-params return <td>{$tds[$pos+1]}</td>}
-                            <td>{$cat}</td>
+                            <td>{$cat-name}</td>
                         </tr>
 
     let $table := <table class="table table-bordered table-light table-hover datatable" id="bulkTable">
         <thead>{$th}</thead>
         {$trs}
         </table>
-    (: Maybe later : let $votable := jmmc-tap:table2votable($table, "targets") :)
+    let $votable := jmmc-tap:table2votable($table, "targets")
 
-    let $error-report := for $cat in $catalogs-to-query
-        let $info := $bulk-search-map($cat)?ranking
+    let $error-report := for $cat in map:keys($bulk-search-map?catalogs)
+        let $cat := $bulk-search-map?catalogs($cat)
+        let $info := $cat?ranking
         let $error := $info?error
         where exists($error)
         return
@@ -702,8 +698,8 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
 
     let $log := util:log("info", "DONE : main table merged")
 
-    let $targets :=<div><h3>{ count($trs) } best proposed configurations for your {count(map:keys($identifiers-map))} targets (top {$max?rank} and non null for each science source { count( $bulk-search-map?*?ranking?scores?*[number(.)>0] ) }/{ count( $bulk-search-map?*?ranking?scores?* ) })
-        <!-- Maybe later : <a class="btn btn-outline-secondary btn-sm" href="data:application/x-votable+xml;base64,{util:base64-encode(serialize($votable))}" type="application/x-votable+xml" download="input.vot">votable</a>&#160; -->
+    let $targets :=<div><h3>{ count($trs) } best proposed configurations for your {count(map:keys($identifiers-map))} targets (top {$max?rank} and non null for each science source { count( $bulk-search-map?catalogs?*?ranking?scores?*[number(.)>0] ) }/{ count( $bulk-search-map?catalogs?*?ranking?scores?* ) })
+        <a class="btn btn-outline-secondary btn-sm" href="data:application/x-votable+xml;base64,{util:base64-encode(serialize($votable))}" type="application/x-votable+xml" download="input.vot">votable</a>&#160;
         </h3>
         {$error-report}
         {$table}
@@ -712,7 +708,7 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
             <div class="p-2"><div class="input-group"><span class="input-group-text">Min score:</span><input type="text" id="min_score" name="min_score" value="{$config?min?score}"/></div></div>
             <div class="p-2"><div class="input-group"><span class="input-group-text">Max rank:</span><input type="text" id="max_rank" name="max_rank" value="{$max?rank}"/></div></div>
             <div class="p-2"><button class="btn btn-primary" type="submit" formaction="modules/aspro.xql">Get my ASPRO2 file 🤩</button></div>
-            <div class="p-2"><button class="btn btn-primary" type="submit" formaction="modules/test.xql">Test this list</button></div>
+            <!-- <div class="p-2"><button class="btn btn-primary" type="submit" formaction="modules/test.xql">Test this list</button></div> -->
         </div>
 
     </div>
@@ -725,10 +721,32 @@ declare function app:searchftt-bulk-list-html($identifiers as xs:string*, $max a
             <small class="d-inline-flex mb-3 px-2 py-1 fw-semibold bg-warning bg-opacity-10 border border-warning border-opacity-10 rounded-2">UT compliancy</small> or
             <small class="d-inline-flex mb-3 px-2 py-1 fw-semibold bg-danger bg-opacity-10 border border-danger border-opacity-10 rounded-2">not compatible / unknown</small>
         </p>
-        , $bulk-search-map?*?html
+        , $bulk-search-map?catalogs?*?html
         )
 };
 
+(: query given catalogs and perform for rank each ft ao compatible combination
+    output structure is a map :
+    - $res?identifiers-map : map {$identifier : id-info}
+    - $res?catalogs : map{ $catalogName :
+            map {
+                "error" : htmlerror
+                "votable":$votable
+                    or
+                "votable":$votable
+                "html" : $html
+                "targets-map" : map {$identifier : id-info}
+                "ranking : map {
+                    "error": $error
+                    "query" : $query
+                    "sciences-idx" : map { $science-id : array{ $pos-idx } }
+                    "input-params" : array { $colnames }
+                    "inputs" : array { $colvalues_of_colnames }
+                    "ftaos" : array { [ft1, ao1], ... [ftn, aon] }
+                    "scores" : array { $scores }
+                    }
+                }
+:)
 declare function app:searchftt-bulk-list($identifiers as xs:string*, $catalogs-to-query as xs:string* ) {
     let $log := util:log("info", "catalogs to query : " || string-join($catalogs-to-query))
     (: Check that we have requested catalog in our conf :)
@@ -756,13 +774,16 @@ declare function app:searchftt-bulk-list($identifiers as xs:string*, $catalogs-t
     (: TODO iterate and merge over chunk of
     let $votables := jmmc-tap:table2votable($table, "targets", 500) :)
 
+    let $catalogs := map:merge((
+        for $cat-name in $catalogs-to-query
+            let $cat := $app:conf?catalogs?*[?cat_name=$cat-name]
+            let $res  := app:bulk-search($votable, $cat)
+            return map:entry($cat-name,$res)
+        ))
+
     let $bulk-search-map :=  map:merge((
         map:entry("identifiers-map",$identifiers-map),
-        map:entry("catalogs",
-        for $cat-name in $catalogs-to-query
-        let $cat := $app:conf?catalogs?*[?cat_name=$cat-name]
-        let $res  := app:bulk-search($votable, $cat )
-        return map:entry($cat-name,$res))))
+        map:entry("catalogs", $catalogs)))
 
     return
         $bulk-search-map

modules/aspro.xql

@@ -5,53 +5,83 @@ import module namespace app="http://exist.jmmc.fr/searchftt/apps/searchftt/templ
 (: Ask for download :)
 let $headers := response:set-header("Content-Disposition",' attachment; filename="SearchFTT.asprox"')
 
+(: Get config :)
+let $config := app:config()
+
 (: Get form inputs :)
 let $identifiers := request:get-parameter("identifiers",())
 let $catalogs := request:get-parameter("catalogs",())
 
 (: compute lists :)
 let $res := app:searchftt-bulk-list($identifiers, $catalogs)
 
-(: res structure :
-    - $res?identifiers-map : map {$identifier : id-info}
-    - $res?catalogs : map{ $catalogName :
-            map {
-                "error" : htmlerror
-                "votable":$votable
-                    or
-                "votable":$votable
-                "html" : $html
-                "targets-map" : map {$identifier : id-info}
-                "ranking : map {
-                    "error": $error
-                    "query" : $query
-                    "sciences-idx" : map { $science-id : array{ $pos-idx } }
-                    "input-params" : array { $colnames }
-                    "inputs" : array { $colvalues_of_colnames }
-                    "ftaos" : array { [ft1, ao1], ... [ftn, aon] }
-                    "scores" : array { $scores }
-                    }
-                }
-:)
-
 (: TODO filter by min_score and ranking :)
 let $sciences-idx := $res?catalogs?*?ranking?sciences-idx
-let $scores := $res?catalogs?*?ranking?score
-let $ftaos := $res?catalogs?*?ranking?ftaos
-let $fts  := for $ftao in $ftaos?* group by $ft := ($ftao?*)[1] return $ft
-let $aos  := for $ftao in $ftaos?* group by $ao := ($ftao?*)[2] return $ao
+let $sciences := distinct-values(for $m in $sciences-idx return map:keys($m))
+
+(:let $ftaos := $res?catalogs?*?ranking?ftaos:)
+let $ftaos := array{
+    for $ranking in $res?catalogs?*?ranking
+    let $scores := $ranking?scores
+    return
+        for $ftao at $pos in $ranking?ftaos?*
+        let $ftao-score := $scores?*[position()=$pos]
+        where  $ftao-score > $config?min?score
+        return $ftao
+}
+
+let $fts-ids  := for $ftao in $ftaos?* group by $ft := $ftao?*[1] return $ft
+let $aos-ids  := for $ftao in $ftaos?* group by $ao := $ftao?*[2] return $ao
+
+let $all-identifiers := distinct-values( ( $sciences, $fts-ids, $aos-ids) )
+let $targets-map := map:merge(($res?catalogs?*?targets-map, $res?identifiers-map)) (: last given map has the highest priority in this implementation :)
 
-let $all-identifiers := distinct-values( ( for $m in $res?*?ranking?sciences-idx return map:keys($m), $fts, $aos) )
+let $targetInfos :=
+    for $science in distinct-values($sciences)
+        let $all-ftaos := array{ for $ranking in $res?catalogs?*?ranking
+            let $science-idx := $ranking?sciences-idx($science)
+            let $science-scores := $ranking?scores[position()=$science-idx]
 
-(: let $identifiers-map := map:merge( $res?identifiers-map , $res($catalogs)?targets-map ) :)
-(: $res($catalogs)?targets-map, $res?identifiers-map :)
+            let $science-ftaos := for $ftao at $pos in $ranking?ftaos?*[position()=$science-idx]
+                let $ftao-score := $science-scores?*[position()=$pos]
+                where  $ftao-score > $config?min?score
+                order by $ftao-score
+                return $ftao
+
+            return $science-ftaos
+        }
+        (: get distinct fts and aos :)
+        let $science-fts  := for $ftao in $all-ftaos?* group by $ft := $ftao?*[1] return $ft
+        let $science-aos  := for $ftao in $all-ftaos?* group by $ao := $ftao?*[2] return $ao
+
+        return
+            <targetInfo>
+                <targetRef>{app:genTargetIds($science)}</targetRef>
+                <groupMembers>
+                    <groupRef>JMMC_AO</groupRef>
+                    <targets>{string-join(app:genTargetIds($science-aos), " ")}</targets>
+                </groupMembers>
+                <groupMembers>
+                    <groupRef>JMMC_FT</groupRef>
+                    <targets>{string-join(app:genTargetIds($science-fts), " ")}</targets>
+                </groupMembers>
+            </targetInfo>
 
 return
-    (:  doc("../templates/GRAV_FT_AO.asprox")/* :)
+
 <a:observationSetting xmlns:a="http://www.jmmc.fr/aspro-oi/0.1" xmlns:tm="http://www.jmmc.fr/jmcs/models/0.1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
     <schemaVersion>2018.04</schemaVersion>
-    <!--<targetVersion>2019.09</targetVersion>-->
+    <!--Remove to propose coordinates refresh (may be part of a form parameter to add it) <targetVersion>2019.09</targetVersion>-->
     <name>SearchFTT generated</name>
+    <description>This file has been generated by SearchFTT for
+    {
+        string-join(
+        for $param in request:get-parameter-names() order by $param
+        return
+            for $v in request:get-parameter($param,()) return $param ||'='||encode-for-uri($v)
+            ,"&amp;")
+    }
+    </description>
     <when>
         <date>{replace(current-date(),"Z","")}</date>
         <nightRestriction>true</nightRestriction>
@@ -74,8 +104,7 @@ return
     {
         for $identifier in $all-identifiers
             let $target-id := app:genTargetIds($identifier)
-            let $target := $res?identifiers-map($identifier)
-            let $target := if($target) then $target else $res?catalogs?*?targets-map($identifier)
+            let $target := $targets-map($identifier)
             let $name := if($target/name/text()) then $target/name/text() else $identifier
             return
                 <target id="{$target-id}">
@@ -129,37 +158,18 @@ return
         </group>
         <groupMembers>
             <groupRef>JMMC_AO</groupRef>
-            <targets>{ string-join(app:genTargetIds($aos), " ") }</targets>
+            <targets>{ string-join(app:genTargetIds($aos-ids), " ") }</targets>
         </groupMembers>
         <groupMembers>
             <groupRef>JMMC_FT</groupRef>
-            <targets>{ string-join(app:genTargetIds($fts), " ") }</targets>
+            <targets>{ string-join(app:genTargetIds($fts-ids), " ") }</targets>
         </groupMembers>
         {
-            for $science in map:keys($sciences-idx)
-            let $target-id := app:genTargetIds($science)
-            let $indices := $sciences-idx($science)
-
-            let $s_fts := $fts[$indices=position()]
-            let $s_aos := $aos[$indices=position()]
-
-            return
-                <targetInfo>
-                    <targetRef>{$target-id}</targetRef>
-                    <groupMembers>
-                        <groupRef>JMMC_AO</groupRef>
-                        <targets>{string-join(app:genTargetIds($s_aos), " ")}</targets>
-                    </groupMembers>
-                    <groupMembers>
-                        <groupRef>JMMC_FT</groupRef>
-                        <targets>{string-join(app:genTargetIds($s_fts), " ")}</targets>
-                    </groupMembers>
-                </targetInfo>
+            $targetInfos
         }
+
     </targetUserInfos>
     <variant>
         <stations>UT1 UT2 UT3 UT4</stations>
     </variant>
-</a:observationSetting>
-
-
+</a:observationSetting>

modules/test.xql

@@ -39,18 +39,26 @@ import module namespace app="http://exist.jmmc.fr/searchftt/apps/searchftt/templ
 
     let $sciences := $res?identifiers-map
     let $scores := $res?catalogs?*?ranking?scores
-    let $targets-map := $res($catalogs)?targets-map
+    let $targets-map := $res?catalogs?*?targets-map
     let $ftaos := $res?catalogs?*?ranking?ftaos
     let $fts  := for $ftao in $ftaos?* group by $ft := ($ftao?*)[1] return $ft
     let $aos  := for $ftao in $ftaos?* group by $ao := ($ftao?*)[2] return $ao
 
+    let $sciences-idx := $res?catalogs?*?ranking?sciences-idx
+    let $distinct-sciences := distinct-values(for $m in $sciences-idx return map:keys($m))
+
     return
         (
             $catalogs,
+            <distinct-sciences/>,
+            serialize($distinct-sciences, map {"method": "adaptive"}),
             <count-scores/>,
-            count($scores),
+            count($scores?*),
+            map:for-each( $res?catalogs, function ($ck, $cv) { " " || $ck || ':' ||count($cv?ranking?scores)}),
             <count-ftaos/>,
-            count($ftaos),
+            count($ftaos?*),
+            map:for-each( $res?catalogs, function ($ck, $cv) { " " || $ck || ':' ||count($cv?ftaos)}),
+
             <scores/>,
             serialize($scores, map {"method": "adaptive"}),
             <ftaos/>,