From 47f9c205c0879b6f65953022d8ad63a1f154a9fe Mon Sep 17 00:00:00 2001
From: luigiba <baldarigigi@gmail.com>
Date: Mon, 9 Sep 2019 12:29:25 +0200
Subject: [PATCH] Dataset statistics

---
 .idea/workspace.xml |   7 +--
 split/generate.py   | 122 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 121 insertions(+), 8 deletions(-)

diff --git a/.idea/workspace.xml b/.idea/workspace.xml
index 6f1588a..590b085 100644
--- a/.idea/workspace.xml
+++ b/.idea/workspace.xml
@@ -1,12 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="ChangeListManager">
-    <list default="true" id="0f79fb12-bf2b-474d-8da4-e622edb018dd" name="Default Changelist" comment="">
-      <change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/colab/run_dbpedia.sh" beforeDir="false" afterPath="$PROJECT_DIR$/colab/run_dbpedia.sh" afterDir="false" />
-      <change beforePath="$PROJECT_DIR$/split/generate.py" beforeDir="false" afterPath="$PROJECT_DIR$/split/generate.py" afterDir="false" />
-    </list>
+    <list default="true" id="0f79fb12-bf2b-474d-8da4-e622edb018dd" name="Default Changelist" comment="" />
     <option name="EXCLUDED_CONVERTED_TO_IGNORED" value="true" />
     <option name="SHOW_DIALOG" value="false" />
     <option name="HIGHLIGHT_CONFLICTS" value="true" />
diff --git a/split/generate.py b/split/generate.py
index 0a966f6..f3b4f5b 100644
--- a/split/generate.py
+++ b/split/generate.py
@@ -9,6 +9,7 @@
     split data into training/test/validation set
     split data into batches
     generate ontology_constrain file
+    print useful statistics
 
 This script will output a directory which is named as n, where n is number of batches used to split the dataset.
 Inside the directory there will be n-1 (named from 0 to n-1) sub-folders. Each sub-folder contains a different batch.
@@ -188,7 +189,8 @@
 BATCH_SIZE = math.floor(n_triples / N_BATCHES)
 
 print(" LOG:\tNumber of triples: {}\n".format(n_triples))
-
+print(" LOG:\tNumber of entities: {}\n".format(len(entities.keys())))
+print(" LOG:\tNumber of relations: {}\n".format(len(relations.keys())))
 
 
 lef = 0
@@ -356,7 +358,10 @@
             r = str(relations[s[1].strip()])
             f.write(h+' '+t+' '+r+'\n')
     
-          
+    
+    
+    
+    
     print()
     lef = rig
     
@@ -368,6 +373,119 @@
         f.write(key + "\t" + str(value) + "\n")
 
 
+###############################################################################
+
+print("\n ===== Relations structure statistics ===== ")
+
+for b in range(N_BATCHES):
+    print(" ===== Batch {} ===== ".format(b))
+    
+    lef = {}
+    rig = {}
+    
+    if b == 0: triple = open( "./"+str(N_BATCHES)+'/'+str(b)+"/train2id.txt", "r")
+    else: triple = open( "./"+str(N_BATCHES)+'/'+str(b)+"/batch2id.txt", "r")
+    
+    if b == 0: valid = open("./"+str(N_BATCHES)+'/'+str(b)+"/valid2id.txt", "r")
+    else: valid = open("./"+str(N_BATCHES)+'/'+str(b)+"/batchValid2id.txt", "r")
+    
+    if b == 0: test = open("./"+str(N_BATCHES)+'/'+str(b)+"/test2id.txt", "r")
+    else: test = open("./"+str(N_BATCHES)+'/'+str(b)+"/batchTest2id.txt", "r")
+    
+    tot = (int)(triple.readline())
+    for i in range(tot):
+    	content = triple.readline()
+    	h,t,r = content.strip().split()
+    	if not (h,r) in lef:
+    		lef[(h,r)] = []
+    	if not (r,t) in rig:
+    		rig[(r,t)] = []
+    	lef[(h,r)].append(t)
+    	rig[(r,t)].append(h)
+    
+    tot = (int)(valid.readline())
+    for i in range(tot):
+    	content = valid.readline()
+    	h,t,r = content.strip().split()
+    	if not (h,r) in lef:
+    		lef[(h,r)] = []
+    	if not (r,t) in rig:
+    		rig[(r,t)] = []
+    	lef[(h,r)].append(t)
+    	rig[(r,t)].append(h)
+    
+    tot = (int)(test.readline())
+    for i in range(tot):
+    	content = test.readline()
+    	h,t,r = content.strip().split()
+    	if not (h,r) in lef:
+    		lef[(h,r)] = []
+    	if not (r,t) in rig:
+    		rig[(r,t)] = []
+    	lef[(h,r)].append(t)
+    	rig[(r,t)].append(h)
+    
+    test.close()
+    valid.close()
+    triple.close()
+    
+    
+    rellef = {}
+    totlef = {}
+    relrig = {}
+    totrig = {}
+    
+    for i in lef:
+    	if not i[1] in rellef:
+    		rellef[i[1]] = 0
+    		totlef[i[1]] = 0
+    	rellef[i[1]] += len(lef[i])
+    	totlef[i[1]] += 1.0
+    
+    for i in rig:
+    	if not i[0] in relrig:
+    		relrig[i[0]] = 0
+    		totrig[i[0]] = 0
+    	relrig[i[0]] += len(rig[i])
+    	totrig[i[0]] += 1.0
+        
+        
+    
+    for file in {"batch2id.txt", "batchTest2id.txt", "batchValid2id.txt"}:
+        if b == 0 and file == "batch2id.txt": file = "train2id.txt"
+        if b == 0 and file == "batchTest2id.txt": file = "test2id.txt"
+        if b == 0 and file == "batchValid2id.txt": file = "valid2id.txt"
+        
+        s11=0
+        s1n=0
+        sn1=0
+        snn=0
+        f = open("./"+str(N_BATCHES)+'/'+str(b)+"/"+file, "r")
+        tot = (int)(f.readline())
+        for i in range(tot):
+        	content = f.readline()
+        	h,t,r = content.strip().split()
+        	rign = rellef[r] / totlef[r]
+        	lefn = relrig[r] / totrig[r]
+        	if (rign <= 1.5 and lefn <= 1.5):
+        		s11+=1
+        	if (rign > 1.5 and lefn <= 1.5):
+        		s1n+=1
+        	if (rign <= 1.5 and lefn > 1.5):
+        		sn1+=1
+        	if (rign > 1.5 and lefn > 1.5):
+        		snn+=1
+        f.close()
+        
+        print(" LOG:\t# of 1-to-1 triples in {}:{}".format(file, s11))
+        print(" LOG:\t# of 1-to-N triples in {}:{}".format(file, s1n))
+        print(" LOG:\t# of N-to-1 triples in {}:{}".format(file, sn1))
+        print(" LOG:\t# of N-to-N triples in {}:{}".format(file, snn))
+        print()
+        
+        
+
+