diff --git a/__pycache__/run.cpython-37.pyc b/__pycache__/run.cpython-37.pyc index d40c844..59db408 100644 Binary files a/__pycache__/run.cpython-37.pyc and b/__pycache__/run.cpython-37.pyc differ diff --git a/__pycache__/run.cpython-38.pyc b/__pycache__/run.cpython-38.pyc new file mode 100644 index 0000000..1c754bd Binary files /dev/null and b/__pycache__/run.cpython-38.pyc differ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0eb70ec..1423208 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -2942,6 +2942,15 @@ "resolved": "https://registry.npmjs.org/@interactjs/utils/-/utils-1.8.4.tgz", "integrity": "sha512-Kg/da0pQSIvEChGL96fYOSzguDc+2zbz061BWnnskq2viZvPnacFIDC2Z+dEZHH+DBMnnLAt2pqLEulxauSQhA==" }, + "@internetarchive/radial-tree": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@internetarchive/radial-tree/-/radial-tree-1.0.0.tgz", + "integrity": "sha512-BdyszClUwBkDCz+Fl3SehGqFYPk03C0yi6VrT0ssEuWmbeYhMNeqe7hIgrNqUncqCYVNZZn5sqAMfedvk7EIeg==", + "requires": { + "d3": "^5.5.0", + "lodash": "^4.17.10" + } + }, "@mapbox/geojson-area": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/@mapbox/geojson-area/-/geojson-area-0.2.2.tgz", diff --git a/frontend/package.json b/frontend/package.json index 177bb3f..7842f06 100755 --- a/frontend/package.json +++ b/frontend/package.json @@ -17,6 +17,7 @@ "@fortawesome/fontawesome-svg-core": "^1.2.27", "@fortawesome/free-solid-svg-icons": "^5.12.1", "@fortawesome/vue-fontawesome": "^0.1.9", + "@internetarchive/radial-tree": "^1.0.0", "@statnett/vue-plotly": "^0.3.2", "@types/d3-drag": "^1.2.3", "@types/node": "^13.7.4", diff --git a/frontend/src/components/DataSetExecController.vue b/frontend/src/components/DataSetExecController.vue deleted file mode 100644 index c428197..0000000 --- a/frontend/src/components/DataSetExecController.vue +++ /dev/null @@ -1,86 +0,0 @@ - - - diff --git a/frontend/src/components/DataSetSlider.vue b/frontend/src/components/DataSetSlider.vue new file mode 100644 index 0000000..95bdb37 --- /dev/null +++ b/frontend/src/components/DataSetSlider.vue @@ -0,0 +1,156 @@ + + + diff --git a/frontend/src/components/DataSpace.vue b/frontend/src/components/DataSpace.vue new file mode 100644 index 0000000..82b1ba9 --- /dev/null +++ b/frontend/src/components/DataSpace.vue @@ -0,0 +1,226 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/components/FeatureSpace1.vue b/frontend/src/components/FeatureSpace1.vue deleted file mode 100644 index fa38b2c..0000000 --- a/frontend/src/components/FeatureSpace1.vue +++ /dev/null @@ -1,684 +0,0 @@ - - - - - \ No newline at end of file diff --git a/frontend/src/components/FeatureSpace3.vue b/frontend/src/components/FeatureSpace3.vue deleted file mode 100644 index 80d10db..0000000 --- a/frontend/src/components/FeatureSpace3.vue +++ /dev/null @@ -1,686 +0,0 @@ - - - - - \ No newline at end of file diff --git a/frontend/src/components/FeatureSpace4.vue b/frontend/src/components/FeatureSpace4.vue deleted file mode 100644 index 21a836c..0000000 --- a/frontend/src/components/FeatureSpace4.vue +++ /dev/null @@ -1,685 +0,0 @@ - - - - - \ No newline at end of file diff --git a/frontend/src/components/FeatureSpace2.vue b/frontend/src/components/FeatureSpaceDetail.vue similarity index 91% rename from frontend/src/components/FeatureSpace2.vue rename to frontend/src/components/FeatureSpaceDetail.vue index 2c400a9..8a4274d 100644 --- a/frontend/src/components/FeatureSpace2.vue +++ b/frontend/src/components/FeatureSpaceDetail.vue @@ -1,6 +1,6 @@ @@ -13,11 +13,12 @@ import * as d3Base from 'd3' const d3 = Object.assign(d3Base) export default { - name: 'FeatureSpace1', + name: 'FeatureSpaceDetail', data () { return { - dataFS2: [], - dataFS2Trans: [], + dataFS: [], + dataFSTrans: [], + quadrantNumber: 5, jsonData: [], corrMatrixComb: [], corrMatrixCombTotal: [], @@ -29,8 +30,9 @@ export default { methods: { initializeNetwork () { - var listofNodes = this.dataFS2[0] - var dataLoc = JSON.parse(this.dataFS2[3]) + var quadrantNumberLocal = this.quadrantNumber + var listofNodes = this.dataFS[0] + var dataLoc = JSON.parse(this.dataFS[quadrantNumberLocal+2]) var pushEachFinal = [] var pushEach var oldVal @@ -43,13 +45,13 @@ export default { var VIFRemainingLoc = [] var MIRemainingLoc = [] - this.dataFS2Trans.forEach(function(element, index) { + this.dataFSTrans.forEach(function(element, index) { var round = element.round - corrMatrixCombLoc.push(Object.values(JSON.parse(round[5]))) - corrMatrixCombTotalLoc.push(Object.values(JSON.parse(round[9]))[0] * 100) - VIFRemainingLoc.push(Object.values(JSON.parse(round[13]))[0]) - MIRemainingLoc.push(JSON.parse(round[17])) - round = JSON.parse(round[1]) + corrMatrixCombLoc.push(Object.values(JSON.parse(round[quadrantNumberLocal*2-1]))) + corrMatrixCombTotalLoc.push(Object.values(JSON.parse(round[quadrantNumberLocal*3-1]))[0] * 100) + VIFRemainingLoc.push(Object.values(JSON.parse(round[quadrantNumberLocal*4-1]))[0]) + MIRemainingLoc.push(JSON.parse(round[quadrantNumberLocal*5-1])) + round = JSON.parse(round[quadrantNumberLocal-1]) oldVal = 0 newVal = 0 outcome = 0 @@ -69,11 +71,11 @@ export default { pushEach.push({keyIns: "round", valueIns: outcome}) var roundLogE = element.roundLogE - corrMatrixCombLoc.push(Object.values(JSON.parse(roundLogE[5]))) - corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLogE[9]))[0] * 100) - VIFRemainingLoc.push(Object.values(JSON.parse(roundLogE[13]))[0]) - MIRemainingLoc.push(JSON.parse(roundLogE[17])) - roundLogE = JSON.parse(roundLogE[1]) + corrMatrixCombLoc.push(Object.values(JSON.parse(roundLogE[quadrantNumberLocal*2-1]))) + corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLogE[quadrantNumberLocal*3-1]))[0] * 100) + VIFRemainingLoc.push(Object.values(JSON.parse(roundLogE[quadrantNumberLocal*4-1]))[0]) + MIRemainingLoc.push(JSON.parse(roundLogE[quadrantNumberLocal*5-1])) + roundLogE = JSON.parse(roundLogE[quadrantNumberLocal-1]) oldVal = 0 newVal = 0 outcome = 0 @@ -92,11 +94,11 @@ export default { pushEach.push({keyIns: "roundLogE", valueIns: outcome}) var roundLog2 = element.roundLog2 - corrMatrixCombLoc.push(Object.values(JSON.parse(roundLog2[5]))) - corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLog2[9]))[0] * 100) - VIFRemainingLoc.push(Object.values(JSON.parse(roundLog2[13]))[0]) - MIRemainingLoc.push(JSON.parse(roundLog2[17])) - roundLog2 = JSON.parse(roundLog2[1]) + corrMatrixCombLoc.push(Object.values(JSON.parse(roundLog2[quadrantNumberLocal*2-1]))) + corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLog2[quadrantNumberLocal*3-1]))[0] * 100) + VIFRemainingLoc.push(Object.values(JSON.parse(roundLog2[quadrantNumberLocal*4-1]))[0]) + MIRemainingLoc.push(JSON.parse(roundLog2[quadrantNumberLocal*5-1])) + roundLog2 = JSON.parse(roundLog2[quadrantNumberLocal-1]) oldVal = 0 newVal = 0 outcome = 0 @@ -115,11 +117,11 @@ export default { pushEach.push({keyIns: "roundLog2", valueIns: outcome}) var roundLog10 = element.roundLog10 - corrMatrixCombLoc.push(Object.values(JSON.parse(roundLog10[5]))) - corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLog10[9]))[0] * 100) - VIFRemainingLoc.push(Object.values(JSON.parse(roundLog10[13]))[0]) - MIRemainingLoc.push(JSON.parse(roundLog10[17])) - roundLog10 = JSON.parse(roundLog10[1]) + corrMatrixCombLoc.push(Object.values(JSON.parse(roundLog10[quadrantNumberLocal*2-1]))) + corrMatrixCombTotalLoc.push(Object.values(JSON.parse(roundLog10[quadrantNumberLocal*3-1]))[0] * 100) + VIFRemainingLoc.push(Object.values(JSON.parse(roundLog10[quadrantNumberLocal*4-1]))[0]) + MIRemainingLoc.push(JSON.parse(roundLog10[quadrantNumberLocal*5-1])) + roundLog10 = JSON.parse(roundLog10[quadrantNumberLocal-1]) oldVal = 0 newVal = 0 outcome = 0 @@ -195,13 +197,12 @@ export default { graphVizualization () { var computeNodesVar = this.computeNodes - - var listofNodes = this.dataFS2[0] - var uniqueTarget = JSON.parse(this.dataFS2[15]) - var corrTarget = JSON.parse(this.dataFS2[7]) - var corrGlob = JSON.parse(this.dataFS2[11]) - var VIFVar = JSON.parse(this.dataFS2[19]) - var MIVar = JSON.parse(this.dataFS2[23]) + var listofNodes = this.dataFS[0] + var corrTarget = JSON.parse(this.dataFS[this.quadrantNumber*2+2]) + var corrGlob = JSON.parse(this.dataFS[this.quadrantNumber*3+2]) + var uniqueTarget = JSON.parse(this.dataFS[this.quadrantNumber*4+2]) + var VIFVar = JSON.parse(this.dataFS[this.quadrantNumber*5+2]) + var MIVar = JSON.parse(this.dataFS[this.quadrantNumber*6+2]) MIVar = MIVar.concat(this.MIRemaining) var colorCateg = d3.scaleOrdinal(d3.schemeAccent) @@ -288,7 +289,7 @@ export default { .domain(d3.ticks([MIMin, MIMax, 10])) .range(['#67001f','#b2182b','#d6604d','#f4a582','#fddbc7','#d1e5f0','#92c5de','#4393c3','#2166ac','#053061']); - var svg = d3.select("#FeatureGraph2"); + var svg = d3.select("#FeatureGraph"); svg.selectAll("*").remove(); var width = 1000; @@ -301,7 +302,7 @@ export default { var dispatch = d3.dispatch('layoutend'); - svg = d3.select("#FeatureGraph2").append("svg") + svg = d3.select("#FeatureGraph").append("svg") .attr("width", width) .attr("height", height); @@ -653,15 +654,17 @@ export default { }, reset () { - var svg = d3.select("#FeatureGraph2"); + var svg = d3.select("#FeatureGraph"); svg.selectAll("*").remove(); }, }, mounted () { - EventBus.$on('quadTrans2', data => { this.dataFS2Trans = data }) - EventBus.$on('quad2', data => { this.dataFS2 = data }) - EventBus.$on('quad2', this.initializeNetwork) + EventBus.$on('quadTrans', data => { this.dataFSTrans = data }) + EventBus.$on('quad', data => { this.dataFS = data }) + EventBus.$on('quad', this.initializeNetwork) EventBus.$on('countNodes1', data => { this.computeNodes = data }) + + EventBus.$on('reset', this.reset) } } diff --git a/frontend/src/components/FeatureSpaceOverview.vue b/frontend/src/components/FeatureSpaceOverview.vue new file mode 100644 index 0000000..298a600 --- /dev/null +++ b/frontend/src/components/FeatureSpaceOverview.vue @@ -0,0 +1,1048 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/components/Heatmap.vue b/frontend/src/components/Heatmap.vue new file mode 100644 index 0000000..9010d88 --- /dev/null +++ b/frontend/src/components/Heatmap.vue @@ -0,0 +1,595 @@ + + + + + \ No newline at end of file diff --git a/frontend/src/components/Main.vue b/frontend/src/components/Main.vue index 80e77a6..7b05b14 100755 --- a/frontend/src/components/Main.vue +++ b/frontend/src/components/Main.vue @@ -4,165 +4,66 @@ - + - Data Sets Manager - - - - - - - - - - - Provenance + Data Space (Sorted by Predicted Probability) - + + + - - - Results + + + + + Feature Space Overview + + + + + + + + + + + Feature Space Detail + - + + - - - - - - - I. Feature Space - Highly Correct Prediction Probability - - - - - - - - - - - II. Feature Space - Slightly Correct Prediction Probability - - - - - - - - - - - - - III. Feature Space - Slightly Wrong Prediction Probability - - - - - - - - - - - IV. Feature Space - Highly Wrong Prediction Probability - - - - - - - - - + + + + + Final Predictive Results + + + + + + + + + @@ -170,12 +71,12 @@ + + \ No newline at end of file diff --git a/frontend/src/components/Slider.vue b/frontend/src/components/Slider.vue deleted file mode 100644 index 53411ad..0000000 --- a/frontend/src/components/Slider.vue +++ /dev/null @@ -1,99 +0,0 @@ - - - diff --git a/run.py b/run.py index 0a4dc4d..22d5790 100644 --- a/run.py +++ b/run.py @@ -456,7 +456,7 @@ def create_global_function(): # result is mean of test_score return np.mean(result['test_score']) -# check this issue later because we are getting the same results +# check this issue later because we are not getting the same results def executeModel(): create_global_function() @@ -474,12 +474,10 @@ def executeModel(): return 'Everything Okay' -def Transformation(quadrant1, quadrant2, quadrant3, quadrant4): +def Transformation(quadrant1, quadrant2, quadrant3, quadrant4, quadrant5): XDataNumeric = XData.select_dtypes(include='number') - numberOfColumns = len(XDataNumeric.columns) - columns = list(XDataNumeric) - decimals = 0 + columns = list(XDataNumeric) global packCorrTransformed packCorrTransformed = [] @@ -491,49 +489,48 @@ def Transformation(quadrant1, quadrant2, quadrant3, quadrant4): d={} XDataNumericCopy = XDataNumeric.copy() XDataNumericCopy[i] = XDataNumericCopy[i].round() - for number in range(1,5): + for number in range(1,6): quadrantVariable = str('quadrant%s' % number) illusion = locals()[quadrantVariable] d["DataRows{0}".format(number)] = XDataNumericCopy.iloc[illusion, :] - dicTransf["round"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], quadrant1, quadrant2, quadrant3, quadrant4, i, count) + dicTransf["round"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], d['DataRows5'], quadrant1, quadrant2, quadrant3, quadrant4, quadrant5, i, count) d={} XDataNumericCopy = XDataNumeric.copy() XDataNumericCopy[i] = np.log(XDataNumericCopy[i]) XDataNumericCopy[i] = XDataNumericCopy[i].round() - for number in range(1,5): + for number in range(1,6): quadrantVariable = str('quadrant%s' % number) illusion = locals()[quadrantVariable] d["DataRows{0}".format(number)] = XDataNumericCopy.iloc[illusion, :] - dicTransf["roundLogE"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], quadrant1, quadrant2, quadrant3, quadrant4, i, count) + dicTransf["roundLogE"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], d['DataRows5'], quadrant1, quadrant2, quadrant3, quadrant4, quadrant5, i, count) d={} XDataNumericCopy = XDataNumeric.copy() XDataNumericCopy[i] = np.log2(XDataNumericCopy[i]) XDataNumericCopy[i] = XDataNumericCopy[i].round() - for number in range(1,5): + for number in range(1,6): quadrantVariable = str('quadrant%s' % number) illusion = locals()[quadrantVariable] d["DataRows{0}".format(number)] = XDataNumericCopy.iloc[illusion, :] - dicTransf["roundLog2"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], quadrant1, quadrant2, quadrant3, quadrant4, i, count) + dicTransf["roundLog2"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], d['DataRows5'], quadrant1, quadrant2, quadrant3, quadrant4, quadrant5, i, count) d={} XDataNumericCopy = XDataNumeric.copy() XDataNumericCopy[i] = np.log10(XDataNumericCopy[i]) XDataNumericCopy[i] = XDataNumericCopy[i].round() - for number in range(1,5): + for number in range(1,6): quadrantVariable = str('quadrant%s' % number) illusion = locals()[quadrantVariable] d["DataRows{0}".format(number)] = XDataNumericCopy.iloc[illusion, :] - dicTransf["roundLog10"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], quadrant1, quadrant2, quadrant3, quadrant4, i, count) + dicTransf["roundLog10"] = NewComputationTransf(d['DataRows1'], d['DataRows2'], d['DataRows3'], d['DataRows4'], d['DataRows5'], quadrant1, quadrant2, quadrant3, quadrant4, quadrant5, i, count) packCorrTransformed.append(dicTransf) return 'Everything Okay' -def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, quadrant2, quadrant3, quadrant4, feature, count): +def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, DataRows5, quadrant1, quadrant2, quadrant3, quadrant4, quadrant5, feature, count): - # Main Features corrMatrix1 = DataRows1.corr() corrMatrix1 = corrMatrix1.abs() corrMatrix2 = DataRows2.corr() @@ -542,31 +539,38 @@ def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, corrMatrix3 = corrMatrix3.abs() corrMatrix4 = DataRows4.corr() corrMatrix4 = corrMatrix4.abs() + corrMatrix5 = DataRows5.corr() + corrMatrix5 = corrMatrix5.abs() corrMatrix1 = corrMatrix1.loc[[feature]] corrMatrix2 = corrMatrix2.loc[[feature]] corrMatrix3 = corrMatrix3.loc[[feature]] corrMatrix4 = corrMatrix4.loc[[feature]] + corrMatrix5 = corrMatrix5.loc[[feature]] DataRows1 = DataRows1.reset_index(drop=True) DataRows2 = DataRows2.reset_index(drop=True) DataRows3 = DataRows3.reset_index(drop=True) DataRows4 = DataRows4.reset_index(drop=True) + DataRows5 = DataRows5.reset_index(drop=True) targetRows1 = [yData[i] for i in quadrant1] targetRows2 = [yData[i] for i in quadrant2] targetRows3 = [yData[i] for i in quadrant3] targetRows4 = [yData[i] for i in quadrant4] + targetRows5 = [yData[i] for i in quadrant5] targetRows1Arr = np.array(targetRows1) targetRows2Arr = np.array(targetRows2) targetRows3Arr = np.array(targetRows3) targetRows4Arr = np.array(targetRows4) + targetRows5Arr = np.array(targetRows5) uniqueTarget1 = unique(targetRows1) uniqueTarget2 = unique(targetRows2) uniqueTarget3 = unique(targetRows3) uniqueTarget4 = unique(targetRows4) + uniqueTarget5 = unique(targetRows5) if (len(targetRows1Arr) > 0): onehotEncoder1 = OneHotEncoder(sparse=False) @@ -668,21 +672,48 @@ def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, VIF4 = pd.Series() MI4List = [] - corrMatrixComb1 = corrMatrixComb1.loc[[feature]] + if (len(targetRows5Arr) > 0): + onehotEncoder5 = OneHotEncoder(sparse=False) + targetRows5Arr = targetRows5Arr.reshape(len(targetRows5Arr), 1) + onehotEncoder5 = onehotEncoder5.fit_transform(targetRows5Arr) + hotEncoderDF5 = pd.DataFrame(onehotEncoder5) + concatDF5 = pd.concat([DataRows5, hotEncoderDF5], axis=1) + corrMatrixComb5 = concatDF5.corr() + corrMatrixComb5 = corrMatrixComb5.abs() + corrMatrixComb5 = corrMatrixComb5.iloc[:,-len(uniqueTarget5):] + X5 = add_constant(DataRows5.dropna()) + VIF5 = pd.Series([variance_inflation_factor(X5.values, i) + for i in range(X5.shape[1])], + index=X5.columns) + VIF5 = VIF5.loc[[feature]] + if (len(targetRows5Arr) > 2): + MI5 = mutual_info_classif(DataRows5, targetRows5Arr) + MI5List = MI5.tolist() + MI5List = MI5List[count] + else: + MI5List = [] + else: + corrMatrixComb5 = pd.DataFrame() + VIF5 = pd.Series() + MI5List = [] + corrMatrixComb1 = corrMatrixComb1.loc[[feature]] corrMatrixComb2 = corrMatrixComb2.loc[[feature]] corrMatrixComb3 = corrMatrixComb3.loc[[feature]] corrMatrixComb4 = corrMatrixComb4.loc[[feature]] + corrMatrixComb5 = corrMatrixComb5.loc[[feature]] targetRows1ArrDF = pd.DataFrame(targetRows1Arr) targetRows2ArrDF = pd.DataFrame(targetRows2Arr) targetRows3ArrDF = pd.DataFrame(targetRows3Arr) targetRows4ArrDF = pd.DataFrame(targetRows4Arr) + targetRows5ArrDF = pd.DataFrame(targetRows5Arr) concatAllDF1 = pd.concat([DataRows1, targetRows1ArrDF], axis=1) concatAllDF2 = pd.concat([DataRows2, targetRows2ArrDF], axis=1) concatAllDF3 = pd.concat([DataRows3, targetRows3ArrDF], axis=1) concatAllDF4 = pd.concat([DataRows4, targetRows4ArrDF], axis=1) + concatAllDF5 = pd.concat([DataRows5, targetRows5ArrDF], axis=1) corrMatrixCombTotal1 = concatAllDF1.corr() corrMatrixCombTotal1 = corrMatrixCombTotal1.abs() @@ -692,6 +723,8 @@ def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, corrMatrixCombTotal3 = corrMatrixCombTotal3.abs() corrMatrixCombTotal4 = concatAllDF4.corr() corrMatrixCombTotal4 = corrMatrixCombTotal4.abs() + corrMatrixCombTotal5 = concatAllDF5.corr() + corrMatrixCombTotal5 = corrMatrixCombTotal5.abs() corrMatrixCombTotal1 = corrMatrixCombTotal1.loc[[feature]] corrMatrixCombTotal1 = corrMatrixCombTotal1.iloc[:,-1] @@ -701,11 +734,14 @@ def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, corrMatrixCombTotal3 = corrMatrixCombTotal3.iloc[:,-1] corrMatrixCombTotal4 = corrMatrixCombTotal4.loc[[feature]] corrMatrixCombTotal4 = corrMatrixCombTotal4.iloc[:,-1] + corrMatrixCombTotal5 = corrMatrixCombTotal5.loc[[feature]] + corrMatrixCombTotal5 = corrMatrixCombTotal5.iloc[:,-1] corrMatrixCombTotal1 = pd.concat([corrMatrixCombTotal1.tail(1)]) corrMatrixCombTotal2 = pd.concat([corrMatrixCombTotal2.tail(1)]) corrMatrixCombTotal3 = pd.concat([corrMatrixCombTotal3.tail(1)]) corrMatrixCombTotal4 = pd.concat([corrMatrixCombTotal4.tail(1)]) + corrMatrixCombTotal5 = pd.concat([corrMatrixCombTotal5.tail(1)]) packCorrLoc = [] @@ -713,26 +749,31 @@ def NewComputationTransf(DataRows1, DataRows2, DataRows3, DataRows4, quadrant1, packCorrLoc.append(corrMatrix2.to_json()) packCorrLoc.append(corrMatrix3.to_json()) packCorrLoc.append(corrMatrix4.to_json()) + packCorrLoc.append(corrMatrix5.to_json()) packCorrLoc.append(corrMatrixComb1.to_json()) packCorrLoc.append(corrMatrixComb2.to_json()) packCorrLoc.append(corrMatrixComb3.to_json()) packCorrLoc.append(corrMatrixComb4.to_json()) + packCorrLoc.append(corrMatrixComb5.to_json()) packCorrLoc.append(corrMatrixCombTotal1.to_json()) packCorrLoc.append(corrMatrixCombTotal2.to_json()) packCorrLoc.append(corrMatrixCombTotal3.to_json()) packCorrLoc.append(corrMatrixCombTotal4.to_json()) + packCorrLoc.append(corrMatrixCombTotal5.to_json()) packCorrLoc.append(VIF1.to_json()) packCorrLoc.append(VIF2.to_json()) packCorrLoc.append(VIF3.to_json()) packCorrLoc.append(VIF4.to_json()) + packCorrLoc.append(VIF5.to_json()) packCorrLoc.append(json.dumps(MI1List)) packCorrLoc.append(json.dumps(MI2List)) packCorrLoc.append(json.dumps(MI3List)) packCorrLoc.append(json.dumps(MI4List)) + packCorrLoc.append(json.dumps(MI5List)) return packCorrLoc @@ -754,6 +795,9 @@ def Seperation(): quadrant2 = [] quadrant3 = [] quadrant4 = [] + quadrant5 = [] + + probabilityPredictions = [] for index, value in enumerate(getCorrectPrediction): if (value > 50 and value > thresholdsPos): @@ -764,15 +808,18 @@ def Seperation(): quadrant3.append(index) else: quadrant4.append(index) + quadrant5.append(index) + probabilityPredictions.append(value) - Transformation(quadrant1, quadrant2, quadrant3, quadrant4) + Transformation(quadrant1, quadrant2, quadrant3, quadrant4, quadrant5) + # Main Features DataRows1 = XData.iloc[quadrant1, :] DataRows2 = XData.iloc[quadrant2, :] DataRows3 = XData.iloc[quadrant3, :] DataRows4 = XData.iloc[quadrant4, :] + DataRows5 = XData.iloc[quadrant5, :] - # Main Features corrMatrix1 = DataRows1.corr() corrMatrix1 = corrMatrix1.abs() corrMatrix2 = DataRows2.corr() @@ -781,27 +828,33 @@ def Seperation(): corrMatrix3 = corrMatrix3.abs() corrMatrix4 = DataRows4.corr() corrMatrix4 = corrMatrix4.abs() + corrMatrix5 = DataRows5.corr() + corrMatrix5 = corrMatrix5.abs() DataRows1 = DataRows1.reset_index(drop=True) DataRows2 = DataRows2.reset_index(drop=True) DataRows3 = DataRows3.reset_index(drop=True) DataRows4 = DataRows4.reset_index(drop=True) + DataRows5 = DataRows5.reset_index(drop=True) targetRows1 = [yData[i] for i in quadrant1] targetRows2 = [yData[i] for i in quadrant2] targetRows3 = [yData[i] for i in quadrant3] targetRows4 = [yData[i] for i in quadrant4] + targetRows5 = [yData[i] for i in quadrant5] targetRows1Arr = np.array(targetRows1) targetRows2Arr = np.array(targetRows2) targetRows3Arr = np.array(targetRows3) targetRows4Arr = np.array(targetRows4) + targetRows5Arr = np.array(targetRows5) uniqueTarget1 = unique(targetRows1) uniqueTarget2 = unique(targetRows2) uniqueTarget3 = unique(targetRows3) uniqueTarget4 = unique(targetRows4) - + uniqueTarget5 = unique(targetRows5) + if (len(targetRows1Arr) > 0): onehotEncoder1 = OneHotEncoder(sparse=False) targetRows1Arr = targetRows1Arr.reshape(len(targetRows1Arr), 1) @@ -894,15 +947,40 @@ def Seperation(): VIF4 = pd.Series() MI4List = [] + if (len(targetRows5Arr) > 0): + onehotEncoder5 = OneHotEncoder(sparse=False) + targetRows5Arr = targetRows5Arr.reshape(len(targetRows5Arr), 1) + onehotEncoder5 = onehotEncoder5.fit_transform(targetRows5Arr) + hotEncoderDF5 = pd.DataFrame(onehotEncoder5) + concatDF5 = pd.concat([DataRows5, hotEncoderDF5], axis=1) + corrMatrixComb5 = concatDF5.corr() + corrMatrixComb5 = corrMatrixComb5.abs() + corrMatrixComb5 = corrMatrixComb5.iloc[:,-len(uniqueTarget5):] + X5 = add_constant(DataRows5.dropna()) + VIF5 = pd.Series([variance_inflation_factor(X5.values, i) + for i in range(X5.shape[1])], + index=X5.columns) + if (len(targetRows5Arr) > 2): + MI5 = mutual_info_classif(DataRows5, targetRows5Arr) + MI5List = MI5.tolist() + else: + MI5List = [] + else: + corrMatrixComb5 = pd.DataFrame() + VIF5 = pd.Series() + MI5List = [] + targetRows1ArrDF = pd.DataFrame(targetRows1Arr) targetRows2ArrDF = pd.DataFrame(targetRows2Arr) targetRows3ArrDF = pd.DataFrame(targetRows3Arr) targetRows4ArrDF = pd.DataFrame(targetRows4Arr) + targetRows5ArrDF = pd.DataFrame(targetRows5Arr) concatAllDF1 = pd.concat([DataRows1, targetRows1ArrDF], axis=1) concatAllDF2 = pd.concat([DataRows2, targetRows2ArrDF], axis=1) concatAllDF3 = pd.concat([DataRows3, targetRows3ArrDF], axis=1) concatAllDF4 = pd.concat([DataRows4, targetRows4ArrDF], axis=1) + concatAllDF5 = pd.concat([DataRows5, targetRows5ArrDF], axis=1) corrMatrixCombTotal1 = concatAllDF1.corr() corrMatrixCombTotal1 = corrMatrixCombTotal1.abs() @@ -912,47 +990,57 @@ def Seperation(): corrMatrixCombTotal3 = corrMatrixCombTotal3.abs() corrMatrixCombTotal4 = concatAllDF4.corr() corrMatrixCombTotal4 = corrMatrixCombTotal4.abs() + corrMatrixCombTotal5 = concatAllDF5.corr() + corrMatrixCombTotal5 = corrMatrixCombTotal5.abs() corrMatrixCombTotal1 = pd.concat([corrMatrixCombTotal1.tail(1)]) corrMatrixCombTotal2 = pd.concat([corrMatrixCombTotal2.tail(1)]) corrMatrixCombTotal3 = pd.concat([corrMatrixCombTotal3.tail(1)]) corrMatrixCombTotal4 = pd.concat([corrMatrixCombTotal4.tail(1)]) + corrMatrixCombTotal5 = pd.concat([corrMatrixCombTotal5.tail(1)]) global packCorr packCorr = [] packCorr.append(list(XData.columns.values.tolist())) packCorr.append(json.dumps(target_names)) + packCorr.append(json.dumps(probabilityPredictions)) packCorr.append(corrMatrix1.to_json()) packCorr.append(corrMatrix2.to_json()) packCorr.append(corrMatrix3.to_json()) packCorr.append(corrMatrix4.to_json()) - + packCorr.append(corrMatrix5.to_json()) + packCorr.append(corrMatrixComb1.to_json()) packCorr.append(corrMatrixComb2.to_json()) packCorr.append(corrMatrixComb3.to_json()) packCorr.append(corrMatrixComb4.to_json()) + packCorr.append(corrMatrixComb5.to_json()) packCorr.append(corrMatrixCombTotal1.to_json()) packCorr.append(corrMatrixCombTotal2.to_json()) packCorr.append(corrMatrixCombTotal3.to_json()) packCorr.append(corrMatrixCombTotal4.to_json()) + packCorr.append(corrMatrixCombTotal5.to_json()) packCorr.append(json.dumps(uniqueTarget1)) packCorr.append(json.dumps(uniqueTarget2)) packCorr.append(json.dumps(uniqueTarget3)) packCorr.append(json.dumps(uniqueTarget4)) + packCorr.append(json.dumps(uniqueTarget5)) packCorr.append(VIF1.to_json()) packCorr.append(VIF2.to_json()) packCorr.append(VIF3.to_json()) packCorr.append(VIF4.to_json()) + packCorr.append(VIF5.to_json()) packCorr.append(json.dumps(MI1List)) packCorr.append(json.dumps(MI2List)) packCorr.append(json.dumps(MI3List)) packCorr.append(json.dumps(MI4List)) + packCorr.append(json.dumps(MI5List)) return 'Everything Okay'