From a92b63b9d8fcffb14e9b167dd30759f4329f834f Mon Sep 17 00:00:00 2001
From: Angelos Chatzimparmpas <angelos.chatzimparmpas@gmail.com>
Date: Fri, 8 Nov 2019 19:10:14 +0100
Subject: [PATCH] fixed optimization

Former-commit-id: d1fc0f1f876226fb0b1e4ae3023b0eaa3f3491cb
---
 __pycache__/tsneGrid.cpython-37.pyc | Bin 7548 -> 9632 bytes
 index.html                          |   2 +-
 js/tsne_vis.js                      |  45 ++++++++-----
 tsneGrid.py                         |  99 +++++++++++++++++++++++++++-
 4 files changed, 128 insertions(+), 18 deletions(-)

diff --git a/__pycache__/tsneGrid.cpython-37.pyc b/__pycache__/tsneGrid.cpython-37.pyc
index 6a6cdb15e77062737c73e3539db3051a419f0890..4fc405686e7e246e8a15c382ba4217218166ed48 100644
GIT binary patch
delta 3502
zcma)9TXP&o6`r11?R{tOt6g1v(`9{;1jfM-V}j$P0v2|HOzd)D#9BRCtt_pyvn^*4
zJy}=QCNf|nGZ1b`v7!<lsBkMkz$+<up~y>K2>V-j;5$7lX^j&=Q~mWh-#LAzyJzO>
z)xSR5zF?k<#ljl?Mt}3y>;E|U(9E-6|6yX8JvY8Q!;Ab-neoH-b$*l|yQ8((wUgF(
zdxjt1Vf+N2Qp~5UF+P1$-)8o&5Ie<Z_$PLBrg1i>6+ijtT{aRkr%~QSGtla<x*wXK
zZ<Zq5=NP=t@uBr`zvJftC%}VFkcXTQ5A#S_cfvf%V?4r5-pAu*zZ2mJp5!T>F8iD)
z&y=DfCc`ZTjI>xrLm#CcyxG#6SXpySw0PsPjGou7&pL_5^D+YKtkX<3PRlUN87C#p
zc3cFcA(>3cqzuTQ49Ub|XooRP#@lhlCKe6yYwumrYJWoe?G;VN+6j@A=8j%U!A|dJ
z$d9!%)R(nV7A+_GMXJS0`LcG8q504dyJs{3IuZ(v--`jMw-~1K_SbYL?PO$zXVi8s
zi$XijGj!YP9k5x&-d1c@u~o1+#lF##dlhWIV&Cn_JqI?g*!RFTcMBf}T~PFgJ;^z+
z1B#thtWVue+h(=dE3~#jw4uh|MIO`si<1^P>|jc!3tD@4N26Sk#lFRfQC_ee62W%Z
z%h_Q&tm$OMAX3yiyhNTUXih>5fla7EBO)TRGSM=eoY=Sf#poWN7YE30=}ti$>{<g`
z3>9)zVao@rUs)Qgys`{fg`Fmab(jN0ZtFGuiY5+iu@-X%#bMPC!S}%*hEM$ne0%}U
zD17P<z&Fqzgirk;_;`@cVffS^!5$rHJP-pk8vFE`dc2qv?MhRQ<TW`=IH);e9{CtI
zzNJy3Jb=7WLgY<s8Hzgu?w|yMo7~bBcLdyF2?SRZN3gUpIW8yUq%4;E)FO@ABo;X)
z#zkyv7%OeD*V!WDgM8>tkO%qjeZx8Gh24j#{=l)W{X5vlyY~C3{+CX4OJI$tprgXU
zo(fY~^@Nzf?T^w;m33^-f^KxV6MNRwqf=>}G1EPr*<x|`&zYB50(i!~oE>6Gx0byY
zPci5Qv_tFYmo5}1-A?vmG;)4^sZw5=TQ4nG?nLhHr|nrJT#GHNRc~3gm@VC~R_dLQ
zRoz&%?2@oL=K1-RwO3UBY^5$b{tK41exdIEF?TgRZJ)zn`#jykzoLe^)BW$eiTpsb
z6S1xO#)`OT;dZ;uWqSi95?<ulwW_F8H!9+rov6pm3H0?&xa%!h>z(*&sWE@6w6bB%
zS5>9>YNh%g#9oF_<yA)08{;wNei5uwbj<x4%1`Xa>)sR+`x#xPr$(J7)>A|IVRb`q
zwXm;Z#@=c}DnN_vM3mWGrN_@vtx$c<?rxMay@5eu)wnQO-C^$S!t>F;VaGK+$HKa4
zxE~ZwK4S!RAM0L*kEK|cML(|Te#2+@^f0naJ%bv^bU!!nB74hyFkpSA6I-=}U0J9<
zhsR@E#h`tL9&(Lfji8TUgFq0xN+1DlayaXLVKDv7D7|-nFj~;Wr0h1V&D6AQEm#%2
zWS7=OWwo+t*$b3He<Nr?G<DET8ty;MLr{8$HAbWc3CKCR)VX%ku#ZaC_Dh{Ug4$7%
z)W?qpIvuuIBQF9XNaC^0u7@EXhT{_<5tfW=T)V3;>IDs|XFw%-nJ6((%|4ae><xW<
zvR}mh@4^9L{;ym}a?s0a{<pazEd;&xOQlfIYrpi<o@{Bb0?IOA1(l`4QipPlL&xGu
zq{AYi`Urdy=_q{aWAI6&P0>s!BMyR8I)Mc0lkiEUQ}C%zL${Jp>p+iM8;%ZPrtOVE
zT%&{#*UU$7S%@{|#gS(cA}<HA2G^8HkO@MN{ZMO4OoPLLskppIVr3bebvfBD^If%O
zwGXh$lt>H6;+|OJ*Am1Uzm*`?g>Dq<g%JB7#KE5EKsVaNHXy7a9O?-VcEi{UL=8m4
zJ<*|VbYJ5mJ<(z8RTd|_9|tMvIa%r)Ro37hH$>I}MGlk6qxeY}yJ%m=7h->v;AMbY
z%bdJqe}i25eC$zz5`gcj{^sG`FKy0Ru@*#StvYpPWo@CfQa=l$w0eV=&R#C+_6-_J
zA0ySw+ML|SYg#y2p3d6VlaWO)viM}AOp&HteloJOJ41U3!D7y)@7}HuEEC)$SRq&?
z_#(kA0y?VfrwFP9uMji{P7^#$uu1SOKqp5s)`X6Ixdb_Dcxr~7upLSnBOqbxB)W3v
zr6Hj>axKbBqVyF{uej%VduVa*$a{@O`c~G~*6lk~=`O*y30^1IBDe>bx4%Q?cL}~n
zaG&4-!S@Na2_6#sfWRgAA;FIb-XQof!A}T&O7Js+pA)<ZaPJq!Ur0XjoR85C)8?u}
z0va4<L5RI?_lVGQ`f*0rI2$*@h|*=cFAbde`r8<^nV7CyRsK;@O*MUJ&!W}opA(@8
zCUnl;#7mF&RKJK9DgxgA1)yknr^PRc*a6TPaXvqX;=6O^FWPI%-rHH<plRz!)B5ap
zfSus{{Avm3{rqNNW=dVxe5Yn+W{OWeY7O3IiMNoXju)GzZNKA&hL5oq+*8BnLg`dA
K|D{wQ%>E5Z?RgOZ

delta 1738
zcmZWp&u<$=6rR~#JNA0L{<Z$?B#sj&b)2?IgDM0Fp%8j$RFG1X$fAne%+^UsoOC@w
zt}=sD90^~5!b(WJ*`h}dMY$kxtHhBjLOt+5fH+l1@MhLcEJ!r(eee6;%zN|3cK_zz
zZ<s6Lu#Vtw$@%%tPcvVe8GQ1?+<lxclgh`+W9%?{w!ci~Np+xLMDRMQES&v{C&H0y
zu*rm_!fM^s+?V;1TaFYuTqPRUh>!bV`H7#8kue@10Ujhlt`nVv8VU~)gM>+ln8ZgS
z4L>(Xl*CA!BpMnI6RRGg1`BmDSfh(A1U@W0@Zm1vW&`mEtZ4f-3$LO(YdqGz!3;oe
z^QCzEbru4+$P+BmH|ZGD7-k6;X8{&uI*Z*44lqV6+BYQ|bN`4Ws%AezqbxE|>M>^a
z;{ya4W<Mc(1Jx~9Nt&XuF6Nf|PQmIx^<?`QO*5s7q1e-p6rSQ~mL`^*+I^Pno5T`@
zny0`fC3_;-q-2}GrX>6ANbV-EX~}*+lDh(IMzYU`xgBxv0?=7W|2C4G0yZbvHOXqS
z9D0l&tZkt7kIaI0pYarh25F3?vZ$XKAdy2Ym`GGGG79vAG|<<*oB<wSp7B&-Wl@L6
zXqKrg!(v^PXGd_3=Fa&%%?sRBc!3s%XuOA^OjeZWX@GJPAwYSFR6qrZ6hPwwLF+F!
z5QUvE{s}q(zJvKB5E>9gAcUWS7=bTAEc`UYD)<?Qg)c)4kHu#p7Jd%ibh7;=O^e+y
zlh5Q6LQ`S_Y$}6TQP@1<m;TRnc%_GA;xr_d7+6T0@2Qe212@CK0$1%RlA8l=mVpIs
zflk8RF0l%mXH~Y)@X1@M4-B|vkxo%V+_Z~7$M-PFlfo%{d&u|~D!)8bweE(HHIOGp
z<SRoNZVxH~b#g?#I#gj6psJu2N7O}6b(a$9<t3O*iI$+n%c3!fuAHN*=V<X9Eu9@F
z6pXR^snx^EH^n#Im1G{P0z65MV@-fRlamo2hTjVKy+ALN_it9F-D2vNdz?BAy$IUP
zqkY?{Q`^0s-Yi~oRv^h)6`B6M`dxdkRST_d?d&#o)(`93wmV4w;{KCLx(AtTrxtST
z*3lllWrO@Gxa}OZXf5g~?;bSiZu4lDKCBrYvrfUcYWgtRu@7sJ{d#-rlltC~z15T_
zMfP`_{~^vZs61X~bi6$tV_qDx4Ml6_#Thb=Ug#~ah3G}bkQq6nB^nz!L&mx+=-n3Z
zCUorGHYf#g*;+`#;VnIWU7Qtc9XP{5N$7112v+XX1sZO1f6v}9o{QH~Qdn0^#a+xT
zhjBpB@T-6KFgK+uI*wygu@uAoF82X`<o=bjuh+u+Hg$HlTkpZ>9lH{6-V!f*C`?lr
zpD<Jy@quuTg?Z?Hm9NI);zQzx7zIqI<aFKR>2dtp*^}uL?0+Qkx^AUh#jEaz<rSYl
HZs_=52><1v

diff --git a/index.html b/index.html
index 6efae10..99f0bd9 100755
--- a/index.html
+++ b/index.html
@@ -166,7 +166,7 @@
               <div class="panel panel-default med-bottomProv" style="margin-top:0.2px; margin-bottom:+14px">
                   <div class="panel-heading">
                     <h2 class="panel-title" style="display:inline-block"  data-toggle="tooltip" data-placement="right" title="Tip: a feature of this tool that supports clusters (and points) exploration. Checking the neighborhood preservation between the entire projection's average and a selection driven by the user.">Projections Provenance</h2>
-                    [Sorting Metric:
+                    <div id="textToChange" style="display:inline-block">[Sorting Metric:</div>
                     <select id="param-SortM-view" name="param-SortM-view" onchange="ReSort(false)">
                         <option value="1" selected>Neighborhood Hit (NH)</option>                
                         <option value="2">Trustworthiness (T)</option>
diff --git a/js/tsne_vis.js b/js/tsne_vis.js
index 6c2cde7..392fe4c 100755
--- a/js/tsne_vis.js
+++ b/js/tsne_vis.js
@@ -28,13 +28,13 @@ var ArrayWithCosts = []; var Iterations = [];
 
 var VisiblePoints = []; 
 
-var sliderTrigger = false; var sliderInsideTrigger = false; var parameters; var SelProjIDS; var SelProjIDSProv; var SelectedProjections = []; var activeProjectionNumber = []; var activeProjectionNumberProv = [];
+var sliderTrigger = false; var sliderInsideTrigger = false; var parameters; var SelProjIDS; var SelProjIDSProv; var SelectedProjections = []; var activeProjectionNumber = []; var activeProjectionNumberProv = []; var globalFlagCheck = true;
 
 // This variable is for the kNN Bar Chart in order to store the first execution.
 var inside = 0; var kValuesLegend = []; var findNearestTable = []; var howManyPoints;
 var maxKNN = 0
 
-var mode = 1; var colors = ['#a6cee3','#fb9a99','#b2df8a','#33a02c','#1f78b4','#e31a1c','#fdbf6f','#ff7f00','#cab2d6','#6a3d9a']; var projections = []; var metricsSorting = []; var dataReceivedFromServer = []; var metrics = [];
+var mode = 1; var colors = ['#a6cee3','#fb9a99','#b2df8a','#33a02c','#1f78b4','#e31a1c','#fdbf6f','#ff7f00','#cab2d6','#6a3d9a']; var projections = []; var metricsSorting = []; var dataReceivedFromServer = []; var dataReceivedFromServerOptimized = []; var metrics = []; var FocusedIDs = [];
 
 var Category; var target_names = []
 
@@ -96,7 +96,7 @@ function fetchVal(callback) {
 
 function OptimizePoints() {
   if (lassoFlag) {
-    varFocusedIDs = []
+    FocusedIDs = []
     for (let i = 0; i < points.length; i++) {
       if (points[i].selected) {
         FocusedIDs[i] = i
@@ -104,9 +104,9 @@ function OptimizePoints() {
     }
 
      // ajax the JSON to the server
-     $.post("http://127.0.0.1:5000/receiver", {'data': JSON.stringify(results_local), 'focus': JSON.stringify(FocusedIDs)}, function(){
-      $.get("http://127.0.0.1:5000/sender", function( data ) {
-        dataReceivedFromServer = data
+     $.post("http://127.0.0.1:5000/receiverOptimizer", JSON.stringify(FocusedIDs), function(){
+      $.get("http://127.0.0.1:5000/senderOptimizer", function( data ) {
+        dataReceivedFromServerOptimized = data
         ReSort(false)
     });
   
@@ -130,10 +130,32 @@ function ReSort(flagInitialize) {
 
   Plotly.purge(graphDiv);
 
+  var metricsSortingCopy
+  var metricsCopy
+
   projections = dataReceivedFromServer['projections']
   parameters = dataReceivedFromServer['parameters']
   metricsSorting = dataReceivedFromServer['metrics']
 
+  if (FocusedIDs.length != 0) {
+    if (globalFlagCheck) {
+      metricsSortingCopy = metricsSorting
+      metricsCopy = metrics
+    }
+    globalFlagCheck = false
+    document.getElementById("textToChange").innerHTML = "[Sorting Metric for Optimized Selection:";
+    metricsSorting = dataReceivedFromServerOptimized['metrics']
+    metrics = dataReceivedFromServerOptimized['metricsEntire']
+    if (FocusedIDs.length == points.length) {
+      document.getElementById("textToChange").innerHTML = "[Sorting Metric:";
+    }
+  } else {
+    if (!globalFlagCheck) {
+      metricsSorting = metricsSortingCopy
+      metrics = metricsCopy
+    }
+  }
+
   var traces = []
   var target_names = []
 
@@ -172,6 +194,7 @@ function ReSort(flagInitialize) {
   labelsTarget = uniqueTarget
 }
 
+
 var optionMetric = document.getElementById("param-SortM-view").value; // Get the threshold value with which the user set's the boundaries of the schema investigation
 var order = [];
 SelectedProjections = []
@@ -188,14 +211,8 @@ if (optionMetric == 1) {
   order = metricsSorting[optionMetric-1]
 }
 
-console.log(order)
-
-console.log(activeProjectionNumber)
-
 var index = order.indexOf(activeProjectionNumber);
 
-console.log(index)
-
 var arrayLineColor = []
 
 for (let k = 0; k < 8; k++) {
@@ -221,8 +238,6 @@ for (let k = 0; k < 8; k++) {
   }
 }
 
-console.log(arrayLineColor)
-
 var checkCounter = 0
 var checkCounterMetr = 0
 
@@ -555,7 +570,7 @@ if(k >= 8) {
       margin: {
         l: 10,
         r: 10,
-        b: 8,
+        b: 2,
         t: 2,
         pad: 0
       },
diff --git a/tsneGrid.py b/tsneGrid.py
index e5d333f..98b5bbe 100644
--- a/tsneGrid.py
+++ b/tsneGrid.py
@@ -145,6 +145,7 @@ def calculateGrid():
     global dataProc
     dataProc, length, labels = preprocess(data)
 
+    global D_highSpace
     D_highSpace = distance.squareform(distance.pdist(dataProc))
 
     DEFAULT_NO_DIMS = 2
@@ -180,6 +181,8 @@ def calculateGrid():
     SelectedListofParams = []
     global SelectedProjectionsReturn
     SelectedProjectionsReturn = []
+    
+    global clusterIndex
     clusterIndex = procrustesFun(projectionsAll)
 
     metricNeigh = []
@@ -187,6 +190,8 @@ def calculateGrid():
     metricCont = []
     metricStress = []
     metricShepCorr = []
+
+    global convertLabels
     convertLabels = []
     for index, label in enumerate(labels):
         if (label == 0):
@@ -206,13 +211,21 @@ def calculateGrid():
         else:
             pass
 
+    global D_lowSpaceList
+    D_lowSpaceList = []
+
+    global KeepKs
+    KeepKs = []
+
     for index in clusterIndex:
         SelectedProjectionsReturn.append(projectionsAll[index].tolist())
         SelectedListofParams.append(listofParamsAll[index])
 
         D_lowSpace = distance.squareform(distance.pdist(projectionsAll[index]))
+        D_lowSpaceList.append(D_lowSpace)
 
         k = listofParamsAll[index][0] # k = perplexity
+        KeepKs.append(k)
 
         #resultNeigh = neighborhood_hit(np.array(projectionsAll[index]), convertLabels, k)
         resultNeigh = trustworthiness(D_highSpace, D_lowSpace, k)
@@ -253,7 +266,7 @@ def calculateGrid():
         valueStress = (metricStress[index] - min_value_stress) / (max_value_stress - min_value_stress) 
         valueShep = (metricShepCorr[index] - min_value_shep) / (max_value_shep - min_value_shep) 
         metricsMatrixEntire.append([valueNeigh,valueTrust,valueCont,valueStress,valueShep])
-    
+
     sortNeigh = sorted(range(len(metricNeigh)), key=lambda k: metricNeigh[k], reverse=True)
     sortTrust = sorted(range(len(metricTrust)), key=lambda k: metricTrust[k], reverse=True)
     sortCont = sorted(range(len(metricCont)), key=lambda k: metricCont[k], reverse=True)
@@ -283,5 +296,87 @@ def background_process():
         pass
     return jsonify({ 'projections': SelectedProjectionsReturn, 'parameters': SelectedListofParams, 'metrics': metricsMatrix, 'metricsEntire': metricsMatrixEntire })
 
+@app.route('/receiverOptimizer', methods = ['POST'])
+def OptimizeSelection():
+    dataReceived= request.get_data().decode('utf8').replace("'", '"')
+    dataReceived = json.loads(dataReceived)
+    dataSelected = []
+    for data in dataReceived:
+        if data != None:
+            dataSelected.append(data)
+
+    metricNeigh = []
+    metricTrust = []
+    metricCont = []
+    metricStress = []
+    metricShepCorr = []
+
+    for index, loop in enumerate(clusterIndex):
+        #resultNeigh = neighborhood_hit(np.array(projectionsAll[index]), convertLabels, k)
+        resultNeigh = trustworthiness(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :], KeepKs[index])
+        resultTrust = trustworthiness(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :], KeepKs[index])
+        resultContinuity = continuity(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :], KeepKs[index])
+        resultStress = normalized_stress(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :])
+        resultShep = normalized_stress(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :])
+        #resultShep = shepard_diagram_correlation(D_highSpace[dataSelected, :], D_lowSpaceList[index][dataSelected, :]) 
+
+        metricNeigh.append(resultNeigh)
+        metricTrust.append(resultTrust)
+        metricCont.append(resultContinuity)
+        metricStress.append(resultStress)
+        metricShepCorr.append(resultShep)
+
+    max_value_neigh = max(metricNeigh)
+    min_value_neigh = min(metricNeigh)
+
+    max_value_trust = max(metricTrust)
+    min_value_trust = min(metricTrust)
+
+    max_value_cont = max(metricCont)
+    min_value_cont = min(metricCont)
+
+    max_value_stress = max(metricStress)
+    min_value_stress = min(metricStress)
+
+    max_value_shep = max(metricShepCorr)
+    min_value_shep = min(metricShepCorr)
+
+    global metricsMatrixEntireSel
+    metricsMatrixEntireSel = []
+
+    for index, data in enumerate(metricTrust):
+        #valueNeigh = (metricNeigh[index] - min_value_neigh) / (max_value_neigh - min_value_neigh) 
+        valueNeigh = (metricTrust[index] - min_value_trust) / (max_value_trust - min_value_trust) 
+        valueTrust = (metricTrust[index] - min_value_trust) / (max_value_trust - min_value_trust) 
+        valueCont = (metricCont[index] - min_value_cont) / (max_value_cont - min_value_cont) 
+        valueStress = (metricStress[index] - min_value_stress) / (max_value_stress - min_value_stress) 
+        valueShep = (metricShepCorr[index] - min_value_shep) / (max_value_shep - min_value_shep) 
+        metricsMatrixEntireSel.append([valueNeigh,valueTrust,valueCont,valueStress,valueShep])
+
+    sortNeigh = sorted(range(len(metricNeigh)), key=lambda k: metricNeigh[k], reverse=True)
+    sortTrust = sorted(range(len(metricTrust)), key=lambda k: metricTrust[k], reverse=True)
+    sortCont = sorted(range(len(metricCont)), key=lambda k: metricCont[k], reverse=True)
+    sortStress = sorted(range(len(metricStress)), key=lambda k: metricStress[k], reverse=True)
+    sortShepCorr = sorted(range(len(metricShepCorr)), key=lambda k: metricShepCorr[k], reverse=True)
+
+    global metricsMatrixSel
+    metricsMatrixSel = []
+
+    metricsMatrixSel.append(sortNeigh)
+    metricsMatrixSel.append(sortTrust)
+    metricsMatrixSel.append(sortCont)
+    metricsMatrixSel.append(sortStress)
+    metricsMatrixSel.append(sortShepCorr)
+
+    return 'OK'
+
+@app.route('/senderOptimizer')
+def SendOptimizedProjections():
+    global metricsMatrixSel
+    global metricsMatrixEntireSel
+
+    return jsonify({'metrics': metricsMatrixSel, 'metricsEntire': metricsMatrixEntireSel })
+
 if __name__ == '__main__':
-    app.run("0.0.0.0", "5000")
\ No newline at end of file
+    app.run("0.0.0.0", "5000")
+