new load

cachedir/joblib/run/GridSearchForModels/7ed3e4ea234ba6e59e062c08100338df/metadata.json

@@ -1 +0,0 @@
-{"duration": 275.9160199165344, "input_args": {"clf": "KNeighborsClassifier(algorithm='ball_tree', leaf_size=30, metric='minkowski',\n                     metric_params=None, n_jobs=None, n_neighbors=24, p=2,\n                     weights='distance')", "params": "{'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24], 'weights': ['uniform', 'distance'], 'algorithm': ['brute', 'kd_tree', 'ball_tree'], 'metric': ['chebyshev', 'manhattan', 'euclidean', 'minkowski']}", "eachAlgor": "'KNN'", "factors": "[1, 1, 1, 1, 1]", "AlgorithmsIDsEnd": "0"}}

cachedir/joblib/run/GridSearchForModels/a071c8e7c3dd03706c3a152c716f8e23/metadata.json

@@ -1 +0,0 @@
-{"duration": 355.3933901786804, "input_args": {"clf": "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='entropy',\n                       max_depth=None, max_features='auto', max_leaf_nodes=None,\n                       min_impurity_decrease=0.0, min_impurity_split=None,\n                       min_samples_leaf=1, min_samples_split=2,\n                       min_weight_fraction_leaf=0.0, n_estimators=119,\n                       n_jobs=None, oob_score=False, random_state=None,\n                       verbose=0, warm_start=False)", "params": "{'n_estimators': [40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119], 'criterion': ['gini', 'entropy']}", "eachAlgor": "'RF'", "factors": "[1, 1, 1, 1, 1]", "AlgorithmsIDsEnd": "576"}}

cachedir/joblib/run/GridSearchForModels/func_code.py

@@ -1,6 +1,6 @@
-# first line: 373
+# first line: 393
 @memory.cache
-def GridSearchForModels(clf, params, eachAlgor, factors, AlgorithmsIDsEnd):
+def GridSearchForModels(XData,yDataclf, params, eachAlgor, factors, AlgorithmsIDsEnd):
 
     # instantiate spark session
     spark = (   

frontend/src/components/DataSetExecController.vue

@@ -1,18 +1,13 @@
 <template>
 <div>
+  <label id="data" for="param-dataset" data-toggle="tooltip" data-placement="right" title="Tip: use one of the data sets already provided or upload a new file.">{{ dataset }}</label>
   <select id="selectFile" @change="selectDataSet()">
       <option value="DiabetesC.csv">Pima Indian Diabetes</option>
       <option value="BreastC.csv">Breast Cancer Winconsin</option>
       <option value="IrisC.csv" selected>Iris</option>
-      <option value="" hidden></option>
+      <option value="local">Upload New File</option>
   </select>
-  <button
-  id="fileInput"
-  type="file"
-  @click="upload"
-  class="mt-2 ml-2">
-  <font-awesome-icon icon="upload"/>
-  Upload</button>
+
   <button
   id="Confirm"
   v-on:click="confirm">
@@ -31,6 +26,11 @@
 <script>
 import Papa from 'papaparse'
 import { EventBus } from '../main.js'
+import {$,jQuery} from 'jquery';
+import * as d3Base from 'd3'
+
+// attach all d3 plugins to the d3 library
+const d3 = Object.assign(d3Base)
 
 export default {
   name: 'DataSetExecController',
@@ -39,17 +39,40 @@ export default {
       RetrieveValueCSV: 'IrisC',
       value: 'Confirm',
       valueReset: 'Reset',
+      dataset: 'Data set'
     }
   },
   methods: {
-    upload () {
-      EventBus.$emit('UploadedFile')
-    },
     selectDataSet () {   
       const fileName = document.getElementById('selectFile')
       this.RetrieveValueCSV = fileName.options[fileName.selectedIndex].value
       this.RetrieveValueCSV = this.RetrieveValueCSV.split('.')[0]
-      EventBus.$emit('SendToServerDataSetConfirmation', this.RetrieveValueCSV)
+
+      if (this.RetrieveValueCSV == "DiabetesC" || this.RetrieveValueCSV == "BreastC" || this.RetrieveValueCSV == "IrisC") { // This is a function that handles a new file, which users can upload.
+        this.dataset = "Data set"
+        d3.select("#data").select("input").remove(); // Remove the selection field.
+        EventBus.$emit('SendToServerDataSetConfirmation', this.RetrieveValueCSV)
+      } else {
+        d3.select("#data").select("input").remove();
+        this.dataset = ""
+        var data
+        d3.select("#data")
+          .append("input")
+          .attr("type", "file")
+          .style("font-size", "16px")
+          .style("width", "200px")
+          .on("change", function() {
+            var file = d3.event.target.files[0];
+            Papa.parse(file, {
+                header: true,
+                dynamicTyping: true,
+                complete: function(results) {
+                  data = results;
+                  EventBus.$emit('SendToServerLocalFile', data)
+                }
+              });
+          })
+      }
     },
     reset () {
       EventBus.$emit('reset')

frontend/src/components/DataSpace.vue

@@ -82,6 +82,9 @@ export default {
       EventBus.$emit('SendProvenance', 'restore')
     },
     scatterPlotDataView () {
+        
+        Plotly.purge('OverviewDataPlotly')
+
         // responsive visualization
         let width = this.responsiveWidthHeight[0]*3 
         let height = this.responsiveWidthHeight[1]*2.1

frontend/src/components/Main.vue

@@ -147,6 +147,7 @@ import { EventBus } from '../main.js'
 import * as jQuery from 'jquery' 
 import $ from 'jquery'
 import * as d3Base from 'd3'
+import Papa from 'papaparse'
 
 // attach all d3 plugins to the d3 library
 const d3 = Object.assign(d3Base)
@@ -181,6 +182,7 @@ export default Vue.extend({
       Collection: 0,
       OverviewResults: 0,
       preDataResults: '',
+      DataResults: '',
       RetrieveValueFile: 'IrisC',
       ClassifierIDsList: '',
       SelectedFeaturesPerClassifier: '',
@@ -216,7 +218,8 @@ export default Vue.extend({
       userSelectedFilterMain: 'mean',
       actionData: '',
       filterData: '',
-      provenanceData: ''
+      provenanceData: '',
+      localFile: ''
     }
   },
   methods: {
@@ -259,6 +262,8 @@ export default Vue.extend({
         .then(response => {
           this.OverviewResults = response.data.OverviewResults
           console.log('Server successfully sent all the data related to visualizations!')
+          console.log(this.OverviewResults)
+          this.DataSpaceCallAfterDataManipulation()
           EventBus.$emit('emittedEventCallingScatterPlot', this.OverviewResults)
           if (this.firstTimeFlag == 1) {
             this.selectedModels_Stack.push(0)
@@ -309,6 +314,10 @@ export default Vue.extend({
           console.log(error)
         })
     },
+    SendToServerData () {
+      // fix that for the upload!
+      console.log(this.localFile)
+    },
     SendSelectedPointsToServer () {
       if (this.ClassifierIDsList === ''){
         this.OverSelLength = 0
@@ -522,6 +531,27 @@ export default Vue.extend({
         .catch(error => {
           console.log(error)
         })
+    },
+      DataSpaceCallAfterDataManipulation () {
+       const path = `http://localhost:5000/data/requestDataSpaceResultsAfterDataManipulation`
+
+      const axiosConfig = {
+        headers: {
+          'Content-Type': 'application/json',
+          'Access-Control-Allow-Origin': '*',
+          'Access-Control-Allow-Headers': 'Origin, Content-Type, X-Auth-Token',
+          'Access-Control-Allow-Methods': 'GET, PUT, POST, DELETE, OPTIONS'
+        }
+      }
+      axios.get(path, axiosConfig)
+        .then(response => {
+          this.DataResults = response.data.DataResults
+          EventBus.$emit('emittedEventCallingDataSpacePlotView', this.DataResults)
+          EventBus.$emit('emittedEventCallingDataPCP', this.DataResults)
+        })
+        .catch(error => {
+          console.log(error)
+        })
     },
     SendAlgorithmsToServer () {
       const path = `http://127.0.0.1:5000/data/ServerRequestSelParameters`
@@ -646,8 +676,6 @@ export default Vue.extend({
           console.log(error)
         })
     },
-    UploadProcess () {
-    },
     render (flag) {
       this.combineWH = []
       this.width = document.body.clientWidth / 12 - 30
@@ -772,6 +800,7 @@ export default Vue.extend({
       axios.post(path, postData, axiosConfig)
         .then(response => {
           console.log('Send request to server! Updated action!')
+          this.getDatafromtheBackEnd()
         })
         .catch(error => {
           console.log(error)
@@ -793,6 +822,9 @@ export default Vue.extend({
       axios.post(path, postData, axiosConfig)
         .then(response => {
           console.log('Send request to server! Updated provenance state!')
+          if (this.provenanceData == 'restore') {
+            this.getDatafromtheBackEnd()
+          }
         })
         .catch(error => {
           console.log(error)
@@ -824,8 +856,6 @@ export default Vue.extend({
     //EventBus.$on('ReturningBrushedPointsIDs',  this.UpdateBarChartFeatures )
     EventBus.$on('ConfirmDataSet', this.fileNameSend)
     EventBus.$on('reset', this.Reset)
-    EventBus.$on('UploadedFile', this.Reset)
-    EventBus.$on('UploadedFile', this.UploadProcess)
     EventBus.$on('ReturningAlgorithms', data => { this.selectedAlgorithms = data })
     EventBus.$on('ReturningBrushedPointsParams', data => { this.parametersofModels = data; })
     EventBus.$on('SendSelectedPointsToServerEvent', data => { this.ClassifierIDsList = data })
@@ -835,6 +865,8 @@ export default Vue.extend({
     EventBus.$on('SendSelectedFeaturesEvent', data => { this.SelectedFeaturesPerClassifier = data })
     EventBus.$on('SendSelectedFeaturesEvent', this.UpdateBasedonFeatures )
     EventBus.$on('SendToServerDataSetConfirmation', data => { this.RetrieveValueFile = data })
+    EventBus.$on('SendToServerLocalFile', data => { this.localFile = data })
+    EventBus.$on('SendToServerLocalFile', this.SendToServerData)
     EventBus.$on('PCPCall', data => { this.selectedAlgorithm = data })
     EventBus.$on('toggle1', data => { this.toggle1 = data })
     EventBus.$on('toggle2', data => { this.toggle2 = data })

frontend/src/components/PCPData.vue

@@ -23,15 +23,15 @@ export default {
   methods: {
     PCPView () {
       d3.selectAll("#PCPDataView > *").remove();
+      const DataSetNew = JSON.parse(this.PCPDataReceived[2])
+      var DataSetParse = JSON.parse(DataSetNew)
       const target_names = JSON.parse(this.PCPDataReceived[3])
-      const DataSet = JSON.parse(this.PCPDataReceived[5])
-      const target = JSON.parse(this.PCPDataReceived[6])
       var colors = this.colorsValues
+      console.log(target_names)
 
       this.pc = ParCoords()("#PCPDataView")
-          .data(DataSet)
+          .data(DataSetParse)
           .color(function(d, i) { return colors[target_names[i]] })
-          .hideAxis([target,'_id','InstanceID'])
           .bundlingStrength(0) // set bundling strength
           .smoothness(0)
           .showControlPoints(false)

frontend/src/components/PredictionsSpace.vue

@@ -18,12 +18,15 @@ export default {
   },
   methods: {
     ScatterPlotDataView () {
+       Plotly.purge('OverviewPredPlotly')
+
       // responsive visualization
       var width = this.responsiveWidthHeight[0]*3
       var height = this.responsiveWidthHeight[1]*1.48 
 
       var target_names = JSON.parse(this.PredictionsData[4])
       const XandYCoordinates = JSON.parse(this.PredictionsData[8])
+      console.log(XandYCoordinates)
       const DataSet = JSON.parse(this.PredictionsData[14])
       const DataSetY = JSON.parse(this.PredictionsData[15])
       const originalDataLabels = JSON.parse(this.PredictionsData[16])

frontend/src/components/ScatterPlot.vue

@@ -54,7 +54,6 @@ export default {
       EventBus.$emit('RemoveFromStack')
     },
     ScatterPlotView () {
-
       Plotly.purge('OverviewPlotly')
       var colorsforScatterPlot = JSON.parse(this.ScatterPlotResults[0])
       var MDSData = JSON.parse(this.ScatterPlotResults[1])

run.py

@@ -63,9 +63,6 @@ def Reset():
     global factors
     factors = [1,1,1,1,1]
 
-    global restoreClicked 
-    restoreClicked = False
-
     global XData
     XData = []
 
@@ -138,9 +135,6 @@ def RetrieveFileName():
     global DataRawLength
     global DataResultsRaw
 
-    global restoreClicked 
-    restoreClicked = False
-
     global RANDOM_SEED
     RANDOM_SEED = 42
 
@@ -235,7 +229,7 @@ def CollectionData():
     }
     return jsonify(response)
 
-def DataSetSelection():  
+def DataSetSelection():
     DataResults = copy.deepcopy(DataResultsRaw)
     for dictionary in DataResultsRaw:
         for key in dictionary.keys():
@@ -246,8 +240,6 @@ def DataSetSelection():
 
     DataResultsRaw.sort(key=lambda x: x[target], reverse=True)
     DataResults.sort(key=lambda x: x[target], reverse=True)
-    
-    dataBefore = copy.deepcopy(DataResults)
 
     for dictionary in DataResults:
         del dictionary['_id']
@@ -255,6 +247,7 @@ def DataSetSelection():
         del dictionary[target]
 
     global AllTargets
+    global target_names
     AllTargets = [o[target] for o in DataResultsRaw]
     AllTargetsFloatValues = []
 
@@ -281,6 +274,30 @@ def DataSetSelection():
     XDataStored = XData.copy()
     yDataStored = yData.copy()
 
+    callPreResults()
+
+    warnings.simplefilter('ignore')
+    return 'Everything is okay'
+
+# Sending each model's results to frontend
+@app.route('/data/requestDataSpaceResultsAfterDataManipulation', methods=["GET", "POST"])
+def SendDataSpaceResultsAfterDataSpaceManipul():
+
+    callPreResults()
+
+    global preResults
+
+    response = {    
+        'DataResults': preResults,
+    }
+    return jsonify(response)
+
+def callPreResults():
+
+    global XData
+    global yData
+    global target_names
+
     DataSpaceRes = FunTsne(XData)
     DataSpaceListRes = DataSpaceRes.tolist()
 
@@ -294,11 +311,6 @@ def DataSetSelection():
     preResults.append(json.dumps(XDataJSONEntireSetRes)) # Position: 2
     preResults.append(json.dumps(yData)) # Position: 3
     preResults.append(json.dumps(AllTargets)) # Position: 4
-    preResults.append(json.dumps(dataBefore)) # Position: 5
-    preResults.append(json.dumps(target)) # Position: 6
-
-    warnings.simplefilter('ignore')
-    return 'Everything is okay'
 
 # Sending each model's results to frontend
 @app.route('/data/requestDataSpaceResults', methods=["GET", "POST"])
@@ -353,6 +365,9 @@ def RetrieveModel():
     global algorithms
     algorithms = RetrievedModel['Algorithms']
 
+    global XData
+    global yData
+
     # loop through the algorithms
     global allParametersPerformancePerModel
     for eachAlgor in algorithms:
@@ -364,7 +379,7 @@ def RetrieveModel():
             clf = RandomForestClassifier()
             params = {'n_estimators': list(range(40, 120)), 'criterion': ['gini', 'entropy']}
             AlgorithmsIDsEnd = 576
-        allParametersPerformancePerModel = GridSearchForModels(clf, params, eachAlgor, factors, AlgorithmsIDsEnd)
+        allParametersPerformancePerModel = GridSearchForModels(XData, yData, clf, params, eachAlgor, factors, AlgorithmsIDsEnd)
 
     # call the function that sends the results to the frontend 
     SendEachClassifiersPerformanceToVisualize()
@@ -376,7 +391,7 @@ memory = Memory(location, verbose=0)
 
 # calculating for all algorithms and models the performance and other results
 @memory.cache
-def GridSearchForModels(clf, params, eachAlgor, factors, AlgorithmsIDsEnd):
+def GridSearchForModels(XData, yData, clf, params, eachAlgor, factors, AlgorithmsIDsEnd):
 
     # instantiate spark session
     spark = (   
@@ -630,6 +645,7 @@ def PreprocessingPred():
     for column, content in df_concatProbs.items():
         el = [sum(x)/len(x) for x in zip(*content)]
         predictions.append(el)
+
     return predictions
 
 def PreprocessingPredUpdate(Models):
@@ -791,16 +807,14 @@ def FunMDS (data):
     return XTransformed
 
 def FunTsne (data):
-    tsne = TSNE(n_components=2).fit_transform(data)
+    tsne = TSNE(n_components=2, random_state=RANDOM_SEED).fit_transform(data)
     tsne.shape
     return tsne
 
 def InitializeEnsemble(): 
-
     XModels = PreprocessingMetrics()
     DataSpace = FunTsne(XData)
     DataSpaceList = DataSpace.tolist()
-
     global ModelSpaceMDS
     global ModelSpaceTSNE
 
@@ -841,7 +855,7 @@ def ReturnResults(ModelSpaceMDS,ModelSpaceTSNE,DataSpaceList,PredictionSpaceList
     featureScoresCon = featureScoresCon.to_json(orient='records')
     XDataJSONEntireSet = XData.to_json(orient='records')
     XDataJSON = XData.columns.tolist()
-
+    print(XData)
     Results.append(json.dumps(sumPerClassifier)) # Position: 0 
     Results.append(json.dumps(ModelSpaceMDS)) # Position: 1
     Results.append(json.dumps(parametersGenPD)) # Position: 2
@@ -1232,7 +1246,6 @@ def EnsembleModel(Models, keyRetrieved):
             temp = json.loads(allParametersPerformancePerModel[1])
             dfParamKNN = pd.DataFrame.from_dict(temp)
             dfParamKNNFilt = dfParamKNN.iloc[:,1]
-            print(featureSelection)
             flag = 0
             for index, eachelem in enumerate(KNNModels):
                 arg = dfParamKNNFilt[eachelem]
@@ -1249,8 +1262,6 @@ def EnsembleModel(Models, keyRetrieved):
                 store = store + 1               
             for index, eachelem in enumerate(RFModels):
                 arg = dfParamRFFilt[eachelem-576]
-                print(index)
-                print(featureSelection['featureSelection'][index+store])
                 all_classifiers.append(make_pipeline(ColumnSelector(cols=featureSelection['featureSelection'][index+store]), RandomForestClassifier().set_params(**arg)))
                 
             sclf = StackingCVClassifier(classifiers=all_classifiers,
@@ -1402,8 +1413,6 @@ def RetrieveAction():
         XData = XData.drop(dataSpacePointsIDs)
         yData = [i for j, i in enumerate(yData) if j not in dataSpacePointsIDs]
 
-    
-
     return 'Done'
 
 # Retrieve data from client 
@@ -1420,12 +1429,12 @@ def RetrieveProvenance():
     global yDataStored
     global yData
 
-    # fix save and restore
-
+    # save and restore
     if (filterProvenanceFinal == 'save'):
         XDataStored = XData
         yDataStored = yData
     else:
         XData = XDataStored.copy()
         yData = yDataStored.copy()
+
     return 'Done'