Can reconstruct document layout from csv

LinkedDataResearch/Client/graph.sparql

@@ -0,0 +1,19 @@
+PREFIX nice: <http://www.semanticweb.org/amitchell/ontologies/nice_all#>
+PREFIX content: <http://www.w3.org/2011/content#>
+PREFIX oa: <http://www.w3.org/ns/oa#>
+PREFIX prov: <http://www.w3.org/ns/prov-o#>
+PREFIX owl: <http://www.w3.org/2002/07/owl#>
+
+construct
+{
+  ?t oa:tag ?tag .
+  ?tag owl:sameAs ?txt .
+}
+where {
+    ?t a nice:Recommendation .
+    ?tgt a oa:SpecificResource .
+    ?tgt oa:hasSource ?t  .
+    ?ann oa:hasTarget ?tgt .
+    ?ann oa:hasBody/content:chars ?txt .
+    ?ann oa:hasBody/owl:sameAs ?tag .
+} 

LinkedDataResearch/Client/queries.js

@@ -39,6 +39,13 @@ module.exports = {
             '}'
         );
     },
+    outcomesForStatement: function(s) {
+        return concat(
+            prefixes,
+            'construct {',
+            '?d content:'
+        );
+    },
     qualityStatementsFor: function(r) {
         return concat(
             prefixes,
@@ -181,17 +188,20 @@ module.exports = {
 
             'construct',
             '{',
+            '?t ',
             '?t oa:tag ?tag .',
             '?tag owl:sameAs ?txt.',
             '}',
             'where {',
+            '?t ',
             '?t nice:isPartOf* ?t1 .',
             '?tgt a oa:SpecificResource .',
             '?tgt oa:hasSource ?t  .',
             '?ann oa:hasTarget ?tgt .',
             '?ann oa:hasBody/content:chars ?txt .',
             '?ann oa:hasBody/owl:sameAs ?tag .',
-            '} LIMIT 500'
-        )
+            'FILTER (REGEX(STR(?t), "CH15", "i"))',
+            '} limit 100'
+        );
     }
 };

LinkedDataResearch/Csv.fs

@@ -156,6 +156,18 @@ module Import =
                           Text = Body q.Question
                           SearchStrategies = loadSearchStrategies q.``Question GUID`` } ]
 
+    let loadSet id  = 
+       sets.Filter(fun r -> r.``Set GUID`` = id).Rows
+        |> Seq.map (fun r -> 
+               { Id = Identifier r.``Set GUID``
+                 Guid = Guid r.``Set GUID``
+                 SetTitle = Title r.``Set rationale``
+                 Discussion = r.``Set discussion``
+                 Questions = loadQuestions r.``Set GUID``
+                 Statements = loadStatements r.``Set GUID``
+                 Rationale = Rationale r.``Set rationale`` })
+        |> Seq.head
+
     let loadSets id = 
         sets.Rows
         |> Seq.map (fun r -> 
@@ -168,8 +180,8 @@ module Import =
                  Rationale = Rationale r.``Set rationale`` })
         |> List.ofSeq
 
-    let loadReccomendations id = 
-        reccomendations.Rows
+    let loadRecommendations id = 
+        reccomendations.Filter(fun r -> r.``Set GUID`` = id).Rows
         |> Seq.map (fun r -> 
                { Id = Identifier r.``Recommendation GUID``
                  Body = Body r.``Recommendation body``
@@ -182,9 +194,13 @@ module Import =
 
     let loadGuidelines = 
         [ for g in guidelines.Rows do
+              let sets = loadSets g.``Guideline GUID``
+              let recs = [for set in sets do 
+                                yield! loadRecommendations (string set.Id)
+                             ]
               yield { Id = Identifier g.``Guideline GUID``
                       Title = Title g.``Guideline Title``
-                      Reccomendation = loadReccomendations id
+                      Reccomendation = recs 
                       Sets = loadSets g.``Guideline GUID``
                       Studies = []
                       Issued = g.``Guideline Publication Date`` } ]

LinkedDataResearch/LinkedDataResearch.fsproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
   <PropertyGroup>
@@ -111,6 +111,7 @@
     <None Include="Serve.fsx" />
     <None Include="packages.config" />
     <None Include="tr.json" />
+    <None Include="Reconstruct.fsx" />
   </ItemGroup>
   <PropertyGroup>
     <MinimumVisualStudioVersion Condition="'$(MinimumVisualStudioVersion)' == ''">11</MinimumVisualStudioVersion>
@@ -125,4 +126,4 @@
   </Target>
   <Target Name="AfterBuild">
   </Target>
-</Project>
+</Project>

LinkedDataResearch/Reconstruct.fsx

@@ -0,0 +1,121 @@
+#r "../packages/ExcelDataReader.2.1/lib/net20/Excel.dll"
+#r "../packages/SharpZipLib.0.86.0/lib/20/ICSharpCode.SharpZipLib.dll"
+#r "../packages/FSharp.Data.2.0.8/lib/net40/FSharp.Data.dll"
+#r "../packages/dotNetRDF.1.0.6-prerelease01/lib/net40/dotNetRDF.dll"
+#r "../packages/FsPickler.0.9.9/lib/net45/FsPickler.dll"
+#r "../packages/FsPickler.Json.0.9.9/lib/net45/FsPickler.Json.dll"
+#r "../packages/openfilesystem.1.0.0.61263243/lib/OpenFileSystem.dll"
+#r "../packages/VDS.Common.1.3.0/lib/net40-client/VDS.Common.dll"
+#r "../OWLTypeProvider.DesignTime/bin/Debug/OWLTypeProvider.DesignTime.dll"
+#load "Excel.fs"
+#load "Model.fs"
+#load "Csv.fs"
+#load "NLP.fs"
+#load "PSeq.fs"
+
+open Import
+open System
+open System.IO
+open System.Data
+open Microsoft.FSharp.Reflection
+open Model
+open FSharp.Data
+open FSharp.Data.JsonExtensions
+open VDS.RDF.Query
+open VDS.RDF
+open Microsoft.FSharp.Collections
+open OpenFileSystem.IO
+open System.Text
+
+let (++) a b = System.IO.Path.Combine(a, b)
+let mkdir (fs : IDirectory) (s:string) = fs.GetDirectory(s.Replace("?","_")).MustExist()
+
+let fixBullets (s:string) =
+    System.Text.RegularExpressions.Regex.Replace(s.Replace("?","\r\n*  "),@"[^\u0000-\u007F]", "")
+
+let write (fs : IDirectory) n (s : string) = 
+    printfn "Writing %s" n
+    use fout = fs.GetFile(n.Replace("?","_")).OpenWrite()
+    fout.Write(System.Text.Encoding.UTF8.GetBytes(s))
+    ()
+
+let lines l = l |> List.fold (fun a i -> a + i + "\r\n") ""
+let directive n v = "" // sprintf "\r\n---\r\n %s: %s\r\n...\r\n" n v
+let citations = System.Collections.Generic.HashSet<string>()
+let h2 s = sprintf "## " + s
+let h3 s = sprintf "### " + s
+
+
+let citeFor s = 
+    printfn "Citation %s" s
+    let k = s + "__bibtex"
+    match NLP.cacheGet k with
+    | Some(x) -> 
+        citations.Add x
+        x
+    | None -> 
+        let r = JsonValue.Parse(Http.RequestString("http://search.labs.crossref.org/dois?q=" + s))
+        let doi = (r.AsArray().[0]?doi).AsString()
+        printfn "Doi : %s" doi
+        if(not(doi.StartsWith "http://dx.doi.org")) then
+             "@article{Unkown_date,}" 
+        else
+            try
+                let cite = Http.RequestString(doi, headers = [ ("Accept", "application/x-bibtex") ])
+                NLP.cacheSet cite k
+                printfn "Got citation %s " cite
+                citations.Add cite
+                cite
+            with 
+            | ex ->  "@article{Unkown_date,}"  
+
+
+let citeId (s : string) = ((s.Split '{').[1].Split ',').[0]
+
+let processCsv() = 
+    let dir = OpenFileSystem.IO.FileSystems.Local.Win32.Win32Directory("c:/temp/")
+    let dir = mkdir dir "guidelines"
+    for g in [Import.loadGuidelines |>List.head] do
+        printfn "%A" g
+        let dir = mkdir dir (string g.Id)
+        let chapters = mkdir dir "chapters"
+        let intro = mkdir chapters "introduction"
+        write intro "Introduction.md" (sprintf "# %s\r\n" (string g.Title))
+        let recs = mkdir chapters "recommendations"
+        for r in g.Reccomendation |> List.filter(fun x -> (string x.Id).StartsWith(string g.Id)) |> Seq.distinctBy (fun r -> (string r.Set)) do
+            printfn "Rec %s" (string r.Id)
+            let recdir = mkdir recs (string ( r.Id))
+            let l = lines [
+                string r.Title
+                (string r.Body) |> fixBullets
+                "'\r\n"
+                directive "nice.evidencegrade" r.Grade 
+            ]
+
+            write recdir "Recommendation.md"  l
+
+            let set = Import.loadSet (string r.Set) 
+            write recdir "Set.md" (lines [
+                h2 ("Topic " + (string set.Id))
+                (string set.SetTitle)
+            ])
+            write recdir "Discussion.md" (lines [
+                set.Discussion |> fixBullets
+            ])
+            for st in set.Statements do
+                let l = lines [
+                             yield h3 ("Evidence Statement " + (string st.Id))
+                             yield string st.Statement |> fixBullets
+                             yield directive "nice.evidencecategory" st.EvidenceCategory
+                             yield ""
+                             yield! [ for study in st.Studies do
+                                          yield ""
+                                          yield "[@" + citeId (citeFor (study.References)) + "]" ] ]
+
+                write recdir ((string st.Id) + ".md") l
+
+        printfn "%A" citations
+        write dir "Citations.bibtex" (lines (citations |> Seq.distinct |> Seq.toList))
+
+do processCsv()
+

LinkedDataResearch/Shiny.r

@@ -0,0 +1,37 @@
+#install.packages(c('SPARQL','igraph','network','ergm'),dependencies=TRUE)
+library(SPARQL)
+library(igraph)
+library(network)
+library(ergm)
+
+endpoint <- "http://localhost:5820/Nice/query"
+sparql_prefix <- "PREFIX content: <http://www.w3.org/2011/content#>
+                  PREFIX oa: <http://www.w3.org/ns/oa#>
+                  PREFIX nice: <http://www.semanticweb.org/amitchell/ontologies/nice_all#>
+                  PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+                  PREFIX xsd: <http://www.w3.org/2001/XMLSchema#>
+"
+
+options <- NULL
+prefix <- c("nice","http://www.semanticweb.org/amitchell/ontologies/nice_all#")
+options <- "output=xml"
+
+q <- paste(sparql_prefix,
+  'select ?t ?txt
+   where {
+    ?t a nice:Recommendation .
+    ?t nice:isPartOf* ?t1 .
+    ?tgt a oa:SpecificResource .
+    ?tgt oa:hasSource ?t  .
+    ?ann oa:hasTarget ?tgt .
+    ?ann oa:hasBody/content:chars ?txt .
+    ?ann oa:hasBody/owl:sameAs ?tag .
+   } limit 1000')
+
+res <- SPARQL(endpoint,q,ns=prefix,extra=options,format="xml")$results
+
+conceptrecmatrix = as.matrix(ifelse(table(res$t,res$txt) > 0, 1, 0))
+
+cr <- graph.incidence(conceptrecmatrix)
+
+write.graph(cr,'concept_rec.graphml',format="graphml")

LinkedDataResearch/Site.fs

@@ -74,10 +74,16 @@ let annotate (store:IStorageProvider) uri content =
     q.SetUri("o",System.Uri(uri))
     q.UpdateProcessor <-  GenericUpdateProcessor(store)
 
+    q.ExecuteUpdate()
+
     let g = new Graph()
 
+
+    printfn "annotating %A %A" uri content
     Project.annotate (Model.Scope(uri,[])) content
-        |> List.iter (fun t -> Store.toStorageTriple g t |> ignore)
+        |> List.iter (fun t -> 
+            printfn "annotate %A" t
+            g.Assert(Store.toStorageTriple g t) |> ignore)
 
     store.UpdateGraph(null :> System.String,g.Triples,[])
 

LinkedDataResearch/ToTriples.fs

@@ -9,7 +9,7 @@ module Project
 
     let annotate (scope:Scope) s = [
         let ex = (NLP.graphOf scope s).Response.Entities |> Array.toList
-
+        printfn "%A" ex
         let rec body (ex:trResponse.Entity list)  i = [
             match ex,i with
             | [],_ -> ()
@@ -222,7 +222,7 @@ module Project
                     yield (a,individual)
                     yield (a,Object.from textContent.Uri)
                     yield (Predicate.from chars.Uri,Object.from (string n.NumeratorDescription))
-                    yield (Predicate.from numerator.ObjectProperties.``nice:hasMeasure``.Uri, Object.from (Owl.Uri(string nscope)))
+                    yield (Predicate.from numerator.ObjectProperties.``nice:isNumeratorOf``.Uri, Object.from (Owl.Uri(string nscope)))
                 ]
             for n in m.Denominators do
              let nscope = scope.Enter(Identifier "denominators").Enter(m.Id)
@@ -233,7 +233,7 @@ module Project
                     yield (a,individual)
                     yield (a,Object.from textContent.Uri)
                     yield (Predicate.from chars.Uri,Object.from (string n.NumeratorDescription))
-                    yield (Predicate.from denominator.ObjectProperties.``nice:hasMeasure``.Uri, Object.from (Owl.Uri(string nscope)))
+                    yield (Predicate.from denominator.ObjectProperties.``nice:isDenominatorOf``.Uri, Object.from (Owl.Uri(string nscope)))
                 ]
     ]
 

LinkedDataResearch/guidelines/CG15/chapters/introduction/Introduction.rst

@@ -0,0 +1 @@
+# Type 1 Diabetes

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.6.1.1.md

@@ -0,0 +1,5 @@
+Diabetes should be confirmed by a single diagnostic laboratory glucose measurement in the presence of classical symptoms, or by a further laboratory glucose measurement. The diagnosis may be supported by a raised HbA1c
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.6.1.2.md

@@ -0,0 +1,9 @@
+Where diabetes is diagnosed, but Type 2 diabetes suspected, the diagnosis of Type 1 diabetes should be considered if:
+?ketonuria is detected, or
+?weight loss is marked, or
+?the person does not have features of the metabolic syndrome or other contributing
+illness.
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.6.1.3.md

@@ -0,0 +1,5 @@
+When diabetes is diagnosed in a younger person, the possibility that the diabetes is not Type 1 diabetes should be considered if they are obese or have a family history of diabetes, particularly if they are of non-white ethnicity. 
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.6.1.4.md

@@ -0,0 +1,5 @@
+Tests to detect specific auto-antibodies or to measure C-peptide deficiency should not be regularly used to confirm the diagnosis of Type 1 diabetes. Their use should be considered if predicting the rate of decline of islet B-cell function would be useful in discriminating Type 1 from Type 2 diabetes. 
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.7.1.1.md

@@ -0,0 +1,5 @@
+Advice to adults with Type 1 diabetes should be provided by a range of professionals with skills in diabetes care working together in a coordinated approach. A common environment (diabetes centre) is an important resource in allowing a diabetes multidisciplinary team to work and communicate efficiently while providing consistent advice.
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.7.1.2.md

@@ -0,0 +1,5 @@
+Open access services should be provided on a walk-in and telephone-request basis during working hours to adults with Type 1 diabetes, and a helpline staffed by people with specific diabetes expertise should be provided on a 24-hour basis. Adults with diabetes should be provided with contact information for these services.
+'
+
+---
+ nice.evidencegrade: C

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.7.1.3.md

@@ -0,0 +1,5 @@
+Each adult with type 1 diabetes should be managed as  an individual, rather than as a member of any cultural, economic or health-affected group. Attention should be paid to the recommendations given elsewhere in this guideline with respect to the cultural preferences of individual adults with type 1 diabetes.
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.7.1.4.md

@@ -0,0 +1,13 @@
+An individual care plan should be set up and reviewed annually, modified according to changes in wishes, circumstances and medical findings, and the details recorded. The plan should include aspects of:
+? diabetes education including nutritional advice (see section 6.1, �Education
+programmes for adults with Type 1 diabetes� and 6.3, �Dietary management�)
+? insulin therapy (see section 7.3, �Insulin regimens� and 7.4, �Insulin delivery�)
+? self-monitoring (see section 6.2, �Self-monitoring of blood glucose�)
+? arterial risk factor surveillance and management (see chapter 8, �Arterial risk control�)
+? late complications surveillance and management (see sections on late complications)
+? means and frequency of communication with the professional care team
+? follow-up consultations including next annual review.
+'
+
+---
+ nice.evidencegrade: D

LinkedDataResearch/guidelines/CG15/chapters/recommendations/CG15R1.7.1.5.md

@@ -0,0 +1,5 @@
+Population, practice-based and clinic diabetes registers (as specified by the national service framework) should be used to assist programmed recall for annual review and assessment of complications and vascular risk.
+'
+
+---
+ nice.evidencegrade: D