mirror of
https://github.com/eclipse-sailing-analytics/sailing-analytics.git
synced 2026-11-12 05:35:31 +00:00
1808 lines
59 KiB
Plaintext
1808 lines
59 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"# Feature selection manoeuvre-based wind estimation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"First the MongoDB loading and pandas configuration code"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"!pip install pymongo"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Initialiaze pandas and co."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"%matplotlib inline\n",
|
|
"import pandas as pd\n",
|
|
"import numpy as np\n",
|
|
"import matplotlib.pyplot as plt\n",
|
|
"import pandas as pd\n",
|
|
"from pymongo import MongoClient\n",
|
|
"from sklearn.feature_selection import SelectPercentile\n",
|
|
"import pathlib\n",
|
|
"from sklearn.metrics import classification_report\n",
|
|
"from sklearn.ensemble import RandomForestClassifier\n",
|
|
"from sklearn.neural_network import MLPClassifier\n",
|
|
"from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis\n",
|
|
"from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
|
|
"from sklearn.linear_model import SGDClassifier\n",
|
|
"from sklearn.linear_model import LogisticRegression\n",
|
|
"from sklearn.svm import LinearSVC\n",
|
|
"from sklearn.naive_bayes import BernoulliNB\n",
|
|
"from sklearn.naive_bayes import GaussianNB\n",
|
|
"from sklearn.neighbors import KNeighborsClassifier\n",
|
|
"from sklearn.neighbors import NearestCentroid\n",
|
|
"from sklearn.ensemble import GradientBoostingClassifier\n",
|
|
"from sklearn.model_selection import GridSearchCV\n",
|
|
"from sklearn.model_selection import cross_val_score\n",
|
|
"from sklearn.pipeline import Pipeline\n",
|
|
"from sklearn.preprocessing import StandardScaler\n",
|
|
"from sklearn.decomposition import PCA\n",
|
|
"import seaborn as sns\n",
|
|
"from sklearn.feature_selection import chi2\n",
|
|
"from sklearn.feature_selection import f_classif"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def _connect_mongo(host, port, username, password, db):\n",
|
|
" if username and password:\n",
|
|
" mongo_uri = 'mongodb://%s:%s@%s:%s/%s' % (username, password, host, port, db)\n",
|
|
" conn = MongoClient(mongo_uri)\n",
|
|
" else:\n",
|
|
" conn = MongoClient(host, port)\n",
|
|
" return conn[db]\n",
|
|
"def read_mongo(db, collection, query={}, host='localhost', port=27017, username=None, password=None, no_id=False):\n",
|
|
" db = _connect_mongo(host=host, port=port, username=username, password=password, db=db)\n",
|
|
" cursor = db[collection].find(query)\n",
|
|
" df = pd.DataFrame(list(cursor))\n",
|
|
" if no_id:\n",
|
|
" del df['_id']\n",
|
|
" return df\n",
|
|
"def init():\n",
|
|
" desired_width = 320\n",
|
|
" pd.set_option('display.width', desired_width)\n",
|
|
" np.set_printoptions(linewidth=desired_width)\n",
|
|
" pd.set_option(\"display.max_columns\", desired_width)\n",
|
|
" pd.set_option(\"display.max_info_rows\", 10000000)\n",
|
|
" plt.figure.max_open_warning = 1000\n",
|
|
" sns.set_style('whitegrid')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"init()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Load manoeuvre data ignoring irrelevant manoeuvre data"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"maneuvers = read_mongo('windEstimation', 'maneuversForDataAnalysis', {'category': {'$ne': 'SMALL'}, 'clean': True, \"deviationTackAngle\":{\"$ne\":None}, \"deviationJibeAngle\":{\"$ne\":None}, 'nextMarkBefore':{\"$ne\":None}, 'nextMarkAfter':{\"$ne\":None}})\n",
|
|
"maneuvers.info()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Convert categorical boat hull type column to multiple binary columns"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"hullType = pd.get_dummies(maneuvers['hullType'], drop_first=True)\n",
|
|
"maneuvers.drop(['hullType'],axis=1,inplace=True)\n",
|
|
"maneuvers = pd.concat([maneuvers, hullType],axis=1)\n",
|
|
"maneuvers.head()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Split manoeuvre data by categories"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"_360 = maneuvers[maneuvers['category'] == '_360']\n",
|
|
"_180 = maneuvers[maneuvers['category'] == '_180']\n",
|
|
"wide = maneuvers[maneuvers['category'] == 'WIDE']\n",
|
|
"regularWithMP = maneuvers[(maneuvers['category'] == 'REGULAR') | (maneuvers['category'] == 'MARK_PASSING')]\n",
|
|
"regularWithoutMP = maneuvers[maneuvers['category'] == 'REGULAR']\n",
|
|
"contenderRegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'Contender']\n",
|
|
"j24RegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'J/24']\n",
|
|
"formula18RegularWithMP = regularWithMP[regularWithMP['boatClass'] == 'Formula 18']\n",
|
|
"contenderRegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'Contender']\n",
|
|
"j24RegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'J/24']\n",
|
|
"formula18RegularWithoutMP = regularWithoutMP[regularWithoutMP['boatClass'] == 'Formula 18']\n",
|
|
"plt.figure()\n",
|
|
"plt.bar(['Regular/MP', 'Regular', 'Wide', '180', '360'], [regularWithMP['_id'].count(), regularWithoutMP['_id'].count(), wide['_id'].count(), _180['_id'].count(), _360['_id'].count()])\n",
|
|
"plt.title('Maneuvers of all boat classes')\n",
|
|
"plt.figure()\n",
|
|
"plt.bar(['Contender', 'J/24', 'Formula 18'], [contenderRegularWithMP['_id'].count(), j24RegularWithMP['_id'].count(), formula18RegularWithMP['_id'].count()])\n",
|
|
"plt.title('Regular WITH mark passing maneuvers of chosen boat classes')\n",
|
|
"plt.figure()\n",
|
|
"plt.bar(['Contender', 'J/24', 'Formula 18'], [contenderRegularWithoutMP['_id'].count(), j24RegularWithoutMP['_id'].count(), formula18RegularWithoutMP['_id'].count()])\n",
|
|
"plt.title('Regular WITHOUT mark passing maneuvers of chosen boat classes')\n",
|
|
"targetColumn = 'type'\n",
|
|
"maneuverTypes = ['TACK', 'JIBE', 'HEAD_UP', 'BEAR_AWAY', 'OTHER', '_180_TACK', '_180_JIBE', '_360']"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"_360['_id'].count()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Prepare datasets for automatic iterations, e.g. for plotting"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"datasets = [regularWithMP, regularWithoutMP, contenderRegularWithMP, j24RegularWithMP, formula18RegularWithMP, contenderRegularWithoutMP, j24RegularWithoutMP, formula18RegularWithoutMP, wide, _180, _360]\n",
|
|
"datasetsTitle = ['Regular/MP', 'Regular', 'Contender RMP', 'J/24 RMP', 'Formula 18 RMP', 'Contender R', 'J/24 R', 'Formula 18 R', 'Wide', '180', '360']\n",
|
|
"def splitDatasetsByType():\n",
|
|
" datasetsSplitByType = []\n",
|
|
" datasetsSplitByTypeLegend = []\n",
|
|
" for dataset in datasets:\n",
|
|
" splitDatasets = {}\n",
|
|
" for maneuverType, subDataset in dataset.groupby(targetColumn):\n",
|
|
" splitDatasets[maneuverType] = subDataset\n",
|
|
" splitDatasetsSorted = []\n",
|
|
" splitDatasetsSortedLegend = []\n",
|
|
" for maneuverType in maneuverTypes:\n",
|
|
" if maneuverType in splitDatasets:\n",
|
|
" splitDatasetsSorted.append(splitDatasets[maneuverType])\n",
|
|
" splitDatasetsSortedLegend.append(maneuverType.replace('_', ''))\n",
|
|
" finalSplitDatasetsSortedLegend = []\n",
|
|
" datasetsSplitByType.append(splitDatasetsSorted)\n",
|
|
" datasetsSplitByTypeLegend.append(splitDatasetsSortedLegend)\n",
|
|
" return (datasetsSplitByType, datasetsSplitByTypeLegend)\n",
|
|
"datasetsSplitByType, datasetsSplitByTypeLegend = splitDatasetsByType()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotCounts(dataset, legend, datasetTitle):\n",
|
|
" plt.figure()\n",
|
|
" plt.bar(legend, [subDataset['maxTurningRate'].count() for subDataset in dataset])\n",
|
|
" plt.title(datasetTitle)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"i = 0\n",
|
|
"for dataset in datasets:\n",
|
|
" plotCounts(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], datasetsTitle[i])\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"_180 = _180[(_180[targetColumn] == '_180_TACK') | (_180[targetColumn] == '_180_JIBE')]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.countplot(x = targetColumn, data = _180, palette = 'rainbow')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Plot all numeric attributes in each dataset as box plot and histogram"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def boxplot(dataset, legend, attribute, datasetTitle, savePath=None):\n",
|
|
" fig = plt.figure()\n",
|
|
" if attribute.startswith('twa'):\n",
|
|
" dataset = [series[attribute].abs() for series in dataset]\n",
|
|
" else:\n",
|
|
" dataset = [series[attribute] for series in dataset]\n",
|
|
" plt.boxplot(dataset, showfliers=False)\n",
|
|
" newLegend = []\n",
|
|
" for i, legendItem in enumerate(legend, 1):\n",
|
|
" newLegend.append(str(i) + ' - ' + legendItem)\n",
|
|
" plt.legend(newLegend)\n",
|
|
" plt.title(datasetTitle + ' - box plot of ' + attribute)\n",
|
|
" if savePath is not None:\n",
|
|
" fig.savefig(savePath)\n",
|
|
" plt.close(fig)\n",
|
|
"def hist(dataset, legend, attribute, datasetTitle, savePath=None):\n",
|
|
" fig = plt.figure()\n",
|
|
" minX = 1\n",
|
|
" maxX = 0\n",
|
|
" for series in dataset:\n",
|
|
" minCandidate = series[attribute].quantile(0.01)\n",
|
|
" maxCandidate = series[attribute].quantile(0.99)\n",
|
|
" minX = min(minX, minCandidate)\n",
|
|
" maxX = max(maxX, maxCandidate)\n",
|
|
" dataset = [series[attribute] for series in dataset]\n",
|
|
" plt.hist(dataset, density=True, histtype='bar', bins=20, range=[minX, maxX])\n",
|
|
" plt.legend(legend)\n",
|
|
" plt.title(datasetTitle + ' - histogram of ' + attribute)\n",
|
|
" if savePath is not None:\n",
|
|
" fig.savefig(savePath)\n",
|
|
" plt.close(fig)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def visualizeData(imgFolderName):\n",
|
|
" pathlib.Path(imgFolderName).mkdir(parents=True, exist_ok=True)\n",
|
|
" for attribute in list(maneuvers):\n",
|
|
" print('## ' + attribute)\n",
|
|
" i = 0\n",
|
|
" for dataset in datasets:\n",
|
|
" if np.issubdtype(dataset[attribute].dtype, np.number):\n",
|
|
" print('##### ' + datasetsTitle[i])\n",
|
|
" dataset.describe()\n",
|
|
" boxplot(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], attribute, datasetsTitle[i], imgFolderName + '/' + attribute + '_boxplot_' + str(i))\n",
|
|
" hist(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], attribute, datasetsTitle[i], imgFolderName + '/' + attribute + '_hist_' + str(i))\n",
|
|
" i += 1\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"visualizeData('dataVisualisation1')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Analyse wide maneuvers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.countplot(x = targetColumn, data = wide, palette = 'rainbow')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.countplot(x = targetColumn, data = _180, palette = 'rainbow')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.countplot(x = targetColumn, data = _360, palette = 'rainbow')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Check TWAs at middle course, lowest speed, max turning rate for wide and circular maneuvers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotTwas():\n",
|
|
" sns.set_style('whitegrid')\n",
|
|
" twaFeatures = ['twaMiddleCourse', 'twaMiddleCourseMainCurve', 'twaLowestSpeed', 'twaMaxTurnRate']\n",
|
|
" twaDatasets = [wide[wide[targetColumn] == '_180_TACK'], wide[wide[targetColumn] == '_180_JIBE'], _360[_360[targetColumn] == '_360']]\n",
|
|
" twaDatasetsTitle = ['Wide tacks', 'Wide jibes', 'Penalty circles']\n",
|
|
" i = 0\n",
|
|
" for dataset in twaDatasets:\n",
|
|
" for twaFeature in twaFeatures:\n",
|
|
" data = [dataset[dataset['starboard'] == False], dataset[dataset['starboard'] == True]]\n",
|
|
" title = twaDatasetsTitle[i] + \" - \" + twaFeature\n",
|
|
" hist(data, ['port', 'starboard'], twaFeature, title)\n",
|
|
" i += 1\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotTwas()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Feature selection"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"irrelevantFeatures = ['type', 'trackId', 'category', '_id', 'clean', 'twaBefore', 'twaAfter', 'tws', 'boatClass', 'fixesCountForPolars', 'twaMiddleCourse', 'twaMiddleCourseMainCurve', 'twaLowestSpeed', 'twaMaxTurnRate', 'starboard', 'hullLength', 'hullBeam', 'SURFERBOARD', 'MONOHULL', 'speedBefore', 'speedAfter']\n",
|
|
"def getInputOutputVectors(maneuvers):\n",
|
|
" inputVector = maneuvers.drop(irrelevantFeatures, axis=1)\n",
|
|
" outputVector = maneuvers['type']\n",
|
|
" inputVectorPositive = inputVector.drop(['deviationJibeAngle', 'deviationTackAngle', 'timeLoss'], axis=1)\n",
|
|
" inputVectorPositive['absDeviationJibeAngle'] = inputVector['deviationJibeAngle'].abs()\n",
|
|
" inputVectorPositive['absDeviationTackAngle'] = inputVector['deviationTackAngle'].abs()\n",
|
|
" inputVectorPositive['absTimeLoss'] = inputVector['timeLoss'].abs()\n",
|
|
" return (inputVector, inputVectorPositive, outputVector)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"from collections import OrderedDict\n",
|
|
"def featureSelectionPlot(maneuvers, scoreFunc, onlyPositive, title):\n",
|
|
" print('#####' + title)\n",
|
|
" inputV, inputVPositive, outputV = getInputOutputVectors(maneuvers)\n",
|
|
" if onlyPositive:\n",
|
|
" inputV = inputVPositive\n",
|
|
" test = SelectPercentile(score_func=scoreFunc, percentile=5)\n",
|
|
" fit = test.fit(inputV, outputV)\n",
|
|
" np.set_printoptions(precision=3)\n",
|
|
" tempScores = {}\n",
|
|
" for i in range(len(inputV.columns)):\n",
|
|
" tempScores[inputV.columns[i]] = fit.scores_[i] / max(fit.scores_)\n",
|
|
" i = 1\n",
|
|
" sortedScores = OrderedDict()\n",
|
|
" for key, value in sorted(tempScores.items(), reverse=True, key=lambda kv: kv[1]):\n",
|
|
" print(str(i) + '. ' + key + \":\\t\" + str(value))\n",
|
|
" sortedScores[key] = value\n",
|
|
" i += 1\n",
|
|
" plt.figure(figsize = (6,6))\n",
|
|
" plt.barh(list(sortedScores.keys())[::-1], list(sortedScores.values())[::-1])\n",
|
|
" plt.title(title)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Feature selection with mutual_info_classif"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def testFeatures(score_func, onlyPositive):\n",
|
|
" i = 0\n",
|
|
" for dataset in datasets:\n",
|
|
" featureSelectionPlot(dataset, score_func, onlyPositive, datasetsTitle[i])\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testFeatures(f_classif, False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Feature selection with chi2 (is a wrong choice for the given data because it is meant to be used for categorical input features)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"#testFeatures(chi2, True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def testFeaturesWithTree():\n",
|
|
" i = 0\n",
|
|
" for dataset in datasets:\n",
|
|
" plotFeatureSelectionWithTree(datasets[i], datasetsTitle[i])\n",
|
|
" i += 1\n",
|
|
"def plotFeatureSelectionWithTree(maneuvers, title):\n",
|
|
" print('#####' + title)\n",
|
|
" inputV, inputVPositive, outputV = getInputOutputVectors(maneuvers)\n",
|
|
" clf = RandomForestClassifier(n_estimators=50)\n",
|
|
" clf = clf.fit(inputV, outputV)\n",
|
|
" tempScores = {}\n",
|
|
" for i in range(len(inputV.columns)):\n",
|
|
" tempScores[inputV.columns[i]] = clf.feature_importances_[i]\n",
|
|
" i = 1\n",
|
|
" sortedScores = OrderedDict()\n",
|
|
" for key, value in sorted(tempScores.items(), reverse=True, key=lambda kv: kv[1]):\n",
|
|
" print(str(i) + '. ' + key + \":\\t\" + str(value))\n",
|
|
" sortedScores[key] = value\n",
|
|
" i += 1\n",
|
|
" plt.figure(figsize=(6,6))\n",
|
|
" plt.barh(list(sortedScores.keys())[::-1], list(sortedScores.values())[::-1])\n",
|
|
" plt.title(title)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testFeaturesWithTree()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Feature correlations"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"i = 0\n",
|
|
"for dataset in datasets:\n",
|
|
" plt.figure(figsize = (6,6))\n",
|
|
" sns.heatmap(dataset.drop(irrelevantFeatures, axis=1).corr(),cmap='coolwarm')\n",
|
|
" plt.title(datasetsTitle[i])\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Check feature individual feature correlations"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": false
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.jointplot(x='speedLossRatio',y='lowestVsExitingSpeedRatio',data=regularWithMP)\n",
|
|
"plt.title(\"Speed loss vs. lowestSpeedVsExitingSpeed in Regular/MP\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"sns.jointplot(x='speedLossRatio',y='maxTurningRate',data=regularWithMP)\n",
|
|
"plt.title(\"Speed loss vs. maxTurningRate in Regular/MP\")"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Head-up and Bear-away maneuvers are very similar. => Merge head ups and bear aways"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def mergeHeadUpsAndBearAways(maneuvers):\n",
|
|
" maneuvers['type'] = maneuvers['type'].apply([lambda mType: 'OTHER' if mType == 'BEAR_AWAY' or mType == 'HEAD_UP' else mType])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for dataset in datasets:\n",
|
|
" mergeHeadUpsAndBearAways(dataset)\n",
|
|
" dataset[targetColumn].unique()\n",
|
|
"datasetsSplitByType, datasetsSplitByTypeLegend = splitDatasetsByType()\n",
|
|
"i = 0\n",
|
|
"for dataset in datasets:\n",
|
|
" plotCounts(datasetsSplitByType[i], datasetsSplitByTypeLegend[i], datasetsTitle[i])\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Visualize data with merged head ups and bear aways"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"visualizeData('dataVisualisation2')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Remove attributes with low relevance"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"finalIrrelevantFeatures = irrelevantFeatures + ['timeLoss', 'speedInOutRatio', 'recoveryPhaseDuration', 'oversteering', 'maneuverDuration', 'mainCurveDuration', 'absMainCurveAngle', 'lowestVsExitingSpeedRatio']\n",
|
|
"finalIrrelevantFeatures"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Construct final datasets"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Define datasets for evaluation"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"polarFeatures = ['deviationTackAngle', 'deviationJibeAngle']\n",
|
|
"markFeatures = ['markPassing']\n",
|
|
"def dropFeatures(dataset, features = []):\n",
|
|
" return dataset.drop(features + finalIrrelevantFeatures[2:], axis=1)\n",
|
|
"evaluationDatasets = [dropFeatures(regularWithMP), dropFeatures(regularWithMP, polarFeatures), dropFeatures(regularWithMP, markFeatures), dropFeatures(regularWithMP, markFeatures + polarFeatures), dropFeatures(contenderRegularWithMP), dropFeatures(j24RegularWithMP), dropFeatures(formula18RegularWithMP), dropFeatures(contenderRegularWithMP, markFeatures), dropFeatures(j24RegularWithMP, markFeatures), dropFeatures(formula18RegularWithMP, markFeatures), dropFeatures(wide, markFeatures + polarFeatures), dropFeatures(_180, markFeatures + polarFeatures)]\n",
|
|
"evaluationDatasetsTitle = ['PolarsMarks', 'Marks', 'Polars', 'Basic', 'Contender Marks', 'J/24 Marks', 'Formula 18 Marks', 'Contender Basic', 'J/24 Basic', 'Formula 18 Basic', 'Wide', '180']\n",
|
|
"i = 0\n",
|
|
"for dataset in evaluationDatasets:\n",
|
|
" print(evaluationDatasetsTitle[i])\n",
|
|
" print(dataset.columns)\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Split datasets into training set and test set. The test set shall contain all manoeuvres recorded within year 2015. All other manoeuvres shall be used for training."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def splitTrainTest(evaluationDatasets, dropFeatures=[]):\n",
|
|
" trainDatasetsInput = []\n",
|
|
" trainDatasetsOutput = []\n",
|
|
" testDatasetsInput = []\n",
|
|
" testDatasetsOutput = []\n",
|
|
" i = 0\n",
|
|
" for dataset in evaluationDatasets:\n",
|
|
" tempDataset = dataset[[targetColumn]]\n",
|
|
" tempDataset['trainTest'] = dataset['trackId'].apply(lambda trackId: 'TEST' if '2018' in trackId else 'TRAIN')\n",
|
|
" trainDataset = dataset[~dataset['trackId'].str.contains(\"2018\")]\n",
|
|
" testDataset = dataset[dataset['trackId'].str.contains(\"2018\")]\n",
|
|
" trainDatasetsInput.append(trainDataset.drop(['trackId', targetColumn] + dropFeatures, axis=1))\n",
|
|
" testDatasetsInput.append(testDataset.drop(['trackId', targetColumn] + dropFeatures, axis=1))\n",
|
|
" trainDatasetsOutput.append(trainDataset[targetColumn])\n",
|
|
" testDatasetsOutput.append(testDataset[targetColumn])\n",
|
|
" plotCounts([trainDataset, testDataset], ['Train', 'Test'], 'Train/test split for ' + evaluationDatasetsTitle[i])\n",
|
|
" plt.figure()\n",
|
|
" sns.countplot(x = targetColumn, hue = 'trainTest', data = tempDataset, palette = 'rainbow')\n",
|
|
" plt.title(evaluationDatasetsTitle[i])\n",
|
|
" i += 1\n",
|
|
" return (trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"(trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput) = splitTrainTest(evaluationDatasets)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"(trainDatasetsOutput[0] == 'TACK').sum()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"i = 0\n",
|
|
"for dataset in trainDatasetsInput:\n",
|
|
" print('# ' + evaluationDatasetsTitle[i])\n",
|
|
" print(dataset.columns)\n",
|
|
" print('#')\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Determine optimal number of components for PCA"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def determineOptimalNumberOfPCAComponents(inputFeatures, label=None):\n",
|
|
" scaler = StandardScaler()\n",
|
|
" scaledInputFeatures = scaler.fit_transform(inputFeatures)\n",
|
|
" pca = PCA()\n",
|
|
" pca.fit(scaledInputFeatures)\n",
|
|
" cumsum = np.cumsum(pca.explained_variance_ratio_)\n",
|
|
" d = np.argmax(cumsum >= 0.95) + 1\n",
|
|
" if label:\n",
|
|
" print(label + \":\\t\" + str(d) + '/' + str(len(inputFeatures.columns)))\n",
|
|
" return d"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"i = 0\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" determineOptimalNumberOfPCAComponents(inputDataset, evaluationDatasetsTitle[i])\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Feature scaling and PCA"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def transformInput(inputFeatures):\n",
|
|
" d = determineOptimalNumberOfPCAComponents(inputFeatures)\n",
|
|
" pipeline = Pipeline([('scaling', StandardScaler()), ('pca', PCA(n_components=d))])\n",
|
|
" transformedInputFeatures = pipeline.fit_transform(inputFeatures)\n",
|
|
" return transformedInputFeatures"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"scaledTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" scaledTrainDatasetsInput.append(StandardScaler().fit_transform(inputDataset))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"transformedTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" transformedTrainDatasetsInput.append(transformInput(inputDataset))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Evaluate various ML models"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def determineBestScoreAndParams(inputFeatures, outputFeatures, clf, paramGrid, label=None):\n",
|
|
" gridSearch = GridSearchCV(clf, paramGrid, cv=3,\n",
|
|
" scoring='f1_macro', return_train_score=True, n_jobs=4)\n",
|
|
" gridSearch.fit(inputFeatures, outputFeatures)\n",
|
|
" if label:\n",
|
|
" print(label + \" - score:\\t\" + str(gridSearch.best_score_))\n",
|
|
" print(str(gridSearch.cv_results_))\n",
|
|
" print('Best params:')\n",
|
|
" print(str(gridSearch.best_params_))\n",
|
|
" print('###########################')\n",
|
|
" else:\n",
|
|
" return gridSearch\n",
|
|
"def crossValidation(clf, inputFeatures=trainDatasetsInput[0], outputFeatures=trainDatasetsOutput[0], label=None):\n",
|
|
" res = cross_val_score(clf, inputFeatures, outputFeatures, cv=3, n_jobs=4, scoring='f1_macro')\n",
|
|
" if label:\n",
|
|
" print(label + \":\\t\" + str(res))\n",
|
|
" return res"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = RandomForestClassifier(n_estimators=100)\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = KNeighborsClassifier(n_neighbors=20)\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = GaussianNB() \n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = BernoulliNB() \n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = LinearSVC(dual=False, C=1) \n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = LogisticRegression()\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = SGDClassifier(penalty='l2', max_iter=5, tol=None)\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = QuadraticDiscriminantAnalysis()\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = LinearDiscriminantAnalysis()\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clf = MLPClassifier(hidden_layer_sizes=(1000, 1000, 100), activation='relu')\n",
|
|
"crossValidation(clf)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Compare classifiers considering it with its assumed optimal hyperparameters (Too many hyperparameter combinations consume a lot of computation time)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"classifiersWithParamsGrid = [\n",
|
|
" (GaussianNB(), [\n",
|
|
" {}\n",
|
|
" ]),\n",
|
|
" (QuadraticDiscriminantAnalysis(), [\n",
|
|
" {}\n",
|
|
" ]),\n",
|
|
" (LinearDiscriminantAnalysis(), [\n",
|
|
" {}\n",
|
|
" #{'solver': ['svd', 'lsqr', 'eigen']}\n",
|
|
" ]),\n",
|
|
" (GradientBoostingClassifier(), [\n",
|
|
" {}\n",
|
|
" ]),\n",
|
|
" (LinearSVC(dual=False, C=1), [\n",
|
|
" {}\n",
|
|
" # {'C': [0.1, 0.5, 1, 10, 100]}\n",
|
|
" ]),\n",
|
|
" (LogisticRegression(), [\n",
|
|
" {}\n",
|
|
" # {'C': [10, 100, 200], 'max_iter': [20, 50, 100]}\n",
|
|
" ]),\n",
|
|
" #(SGDClassifier(tol=None), [\n",
|
|
" # {}\n",
|
|
" # {'penalty': ['l1', 'l2', 'elasticnet'], 'max_iter': [3, 5, 10, 20]}\n",
|
|
" #]),\n",
|
|
" (RandomForestClassifier(n_estimators=30), [\n",
|
|
" {}\n",
|
|
" # {'n_estimators': [30], 'max_features': [4], 'max_depth': [None]}\n",
|
|
" ]),\n",
|
|
" (KNeighborsClassifier(n_neighbors=25), [\n",
|
|
" {}\n",
|
|
" # {'n_neighbors': [15, 20, 25, 30]}\n",
|
|
" ]),\n",
|
|
" (MLPClassifier(hidden_layer_sizes=(100, 100), activation='relu'), [\n",
|
|
" {}\n",
|
|
" # {'hidden_layer_sizes': [(100), (100, 100), (10, 10), (100, 10), (100, 100, 10)], 'activation': ['logistic', 'tanh', 'relu']}\n",
|
|
" ])\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Test classifiers on evaluation datasets"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def testClassifiers(classifiersWithParamsGrid, filterDatasetsWithMaxElements=None):\n",
|
|
" testScores = []\n",
|
|
" trainScores = []\n",
|
|
" fitTimes = []\n",
|
|
" scoreTimes = []\n",
|
|
" clfNames = []\n",
|
|
" bestParams = []\n",
|
|
" for clfWithParamsGrid in classifiersWithParamsGrid:\n",
|
|
" clf = clfWithParamsGrid[0]\n",
|
|
" paramsGrid = clfWithParamsGrid[1]\n",
|
|
" clfName = clf.__class__.__name__\n",
|
|
" clfNames.append(clfName)\n",
|
|
" clfTestScores = []\n",
|
|
" clfTrainScores = []\n",
|
|
" clfFitTimes = []\n",
|
|
" clfScoreTimes = []\n",
|
|
" clfBestParams = []\n",
|
|
" testScores.append(clfTestScores)\n",
|
|
" trainScores.append(clfTrainScores)\n",
|
|
" fitTimes.append(clfFitTimes)\n",
|
|
" scoreTimes.append(clfScoreTimes)\n",
|
|
" bestParams.append(clfBestParams)\n",
|
|
" print('########################')\n",
|
|
" print('Classifier: ' + clfName)\n",
|
|
" i = 0\n",
|
|
" for normal, scaled, transformed in zip(trainDatasetsInput, scaledTrainDatasetsInput, transformedTrainDatasetsInput):\n",
|
|
" outputFeatures = trainDatasetsOutput[i]\n",
|
|
" if filterDatasetsWithMaxElements and len(outputFeatures) > filterDatasetsWithMaxElements:\n",
|
|
" print('Skipping Dataset ' + evaluationDatasetsTitle[i] + ' with ' + str(len(outputFeatures)) + ' elements')\n",
|
|
" i += 1\n",
|
|
" continue\n",
|
|
" print('Dataset ' + evaluationDatasetsTitle[i])\n",
|
|
" for inputFeatures, inputFeaturesTitle in [(normal, 'Normal'), (scaled, 'Scaled'), (transformed, 'Transformed')]:\n",
|
|
" print(inputFeaturesTitle)\n",
|
|
" gridSearch = determineBestScoreAndParams(inputFeatures, outputFeatures, clf, paramsGrid)\n",
|
|
" scores = gridSearch.cv_results_\n",
|
|
" testScore = max(scores['mean_test_score'])\n",
|
|
" j = np.argmax(scores['mean_test_score'])\n",
|
|
" trainScore = scores['mean_train_score'][j]\n",
|
|
" fitTime = scores['mean_fit_time'][j]\n",
|
|
" scoreTime = scores['mean_score_time'][j]\n",
|
|
" clfTestScores.append(testScore)\n",
|
|
" clfTrainScores.append(trainScore)\n",
|
|
" clfFitTimes.append(fitTime)\n",
|
|
" clfScoreTimes.append(scoreTime)\n",
|
|
" clfBestParams.append(gridSearch.best_params_)\n",
|
|
" print(\"testScore:\\t\" + str(testScore))\n",
|
|
" print(\"trainScore:\\t\" + str(trainScore))\n",
|
|
" print(\"fitTime:\\t\" + str(fitTime))\n",
|
|
" print(\"scoreTime:\\t\" + str(scoreTime))\n",
|
|
" print(\"best_params_:\\n\" + str(gridSearch.best_params_))\n",
|
|
" print(\"mean_test_scores:\" + str(scores['mean_test_score']))\n",
|
|
" print(\"mean_test_scores:\" + str(scores['mean_train_score']))\n",
|
|
" print(\"mean_fit_times:\" + str(scores['mean_fit_time']))\n",
|
|
" print(\"mean_score_times:\" + str(scores['mean_score_time']))\n",
|
|
" i += 1\n",
|
|
" return (testScores, trainScores, fitTimes, scoreTimes, clfNames, bestParams)\n"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"scores = testClassifiers(classifiersWithParamsGrid)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"(trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput) = splitTrainTest(evaluationDatasets, dropFeatures=['scaledSpeedBefore', 'scaledSpeedAfter'])\n",
|
|
"scaledTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" scaledTrainDatasetsInput.append(StandardScaler().fit_transform(inputDataset))\n",
|
|
"transformedTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" transformedTrainDatasetsInput.append(transformInput(inputDataset))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {
|
|
"scrolled": true
|
|
},
|
|
"outputs": [],
|
|
"source": [
|
|
"i = 0\n",
|
|
"for dataset in trainDatasetsInput:\n",
|
|
" print('# ' + evaluationDatasetsTitle[i])\n",
|
|
" print(dataset.columns)\n",
|
|
" print('#')\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"scoresWithoutScaledSpeed = testClassifiers(classifiersWithParamsGrid)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Plot the classifier statistics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotScores(scores, testScores, clfNames, label, scoringWithMax):\n",
|
|
" print('########')\n",
|
|
" print(label)\n",
|
|
" tempScores = []\n",
|
|
" sortedScores = []\n",
|
|
" for i in range(int(len(testScores[0]) / 3)):\n",
|
|
" tempScores.append({})\n",
|
|
" sortedScores.append(OrderedDict())\n",
|
|
" for clfName, clfIndex in zip(clfNames, range(len(clfNames))):\n",
|
|
" for i in range(int(len(testScores[0]) / 3)):\n",
|
|
" testScoresToCheck = []\n",
|
|
" for k in range(2):\n",
|
|
" testScoresToCheck.append(testScores[clfIndex][i * 3 + k])\n",
|
|
" bestArg = np.argmax(testScoresToCheck) if scoringWithMax else np.argmin(testScoresToCheck)\n",
|
|
" tempScores[i][clfName] = scores[clfIndex][i * 3 + bestArg]\n",
|
|
" rev = True if scoringWithMax else False\n",
|
|
" revIndex = -1 if scoringWithMax else 1\n",
|
|
" for i in range(int(len(testScores[0]) / 3)):\n",
|
|
" print('# ' + evaluationDatasetsTitle[i])\n",
|
|
" num = 1\n",
|
|
" for key, value in sorted(tempScores[i].items(), reverse=rev, key=lambda kv: kv[1]):\n",
|
|
" print(str(num) + '. ' + key + \":\\t\" + str(value))\n",
|
|
" sortedScores[i][key] = value\n",
|
|
" num += 1\n",
|
|
" fig = plt.figure()\n",
|
|
" plt.barh(list(sortedScores[i].keys())[::revIndex], list(sortedScores[i].values())[::revIndex])\n",
|
|
" plt.title(label + ' - ' + evaluationDatasetsTitle[i])\n",
|
|
" #plt.close(fig)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scores[0], scores[0], scores[4], 'TestScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'TestScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scores[1], scores[0], scores[4], 'TrainScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scoresWithoutScaledSpeed[1], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'TrainScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scores[2], scores[0], scores[4], 'FitTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scoresWithoutScaledSpeed[2], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'FitTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scores[3], scores[0], scores[4], 'ScoreTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotScores(scoresWithoutScaledSpeed[3], scoresWithoutScaledSpeed[0], scoresWithoutScaledSpeed[4], 'ScoreTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Check classifiers performance on transformed, scaled and original datasets"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotTransformationScores(scores, testScores, clfNames, label, scoringWithMax):\n",
|
|
" print('#################################')\n",
|
|
" print(label)\n",
|
|
" rev = True if scoringWithMax else False\n",
|
|
" revIndex = -1 if scoringWithMax else 1\n",
|
|
" legend = ['Normal', 'Scaled', 'Transformed']\n",
|
|
" bestArgs = []\n",
|
|
" tempScores = {}\n",
|
|
" sortedScores = {}\n",
|
|
" for clfName, clfIndex in zip(clfNames, range(len(clfNames))):\n",
|
|
" tempScores[clfName] = []\n",
|
|
" sortedScores[clfName] = []\n",
|
|
" for i in range(int(len(testScores[0]) / 3)):\n",
|
|
" tempScores[clfName].append({})\n",
|
|
" sortedScores[clfName].append(OrderedDict())\n",
|
|
" for k in range(3):\n",
|
|
" tempScores[clfName][i][legend[k]] = scores[clfIndex][i * 3 + k]\n",
|
|
" for clfName in clfNames:\n",
|
|
" print('########### ' + clfName)\n",
|
|
" for i in range(int(len(testScores[0]) / 3)):\n",
|
|
" print('# ' + evaluationDatasetsTitle[i])\n",
|
|
" num = 1\n",
|
|
" for key, value in sorted(tempScores[clfName][i].items(), reverse=rev, key=lambda kv: kv[1]):\n",
|
|
" print(str(num) + '. ' + key + \":\\t\" + str(value))\n",
|
|
" sortedScores[clfName][i][key] = value\n",
|
|
" num += 1\n",
|
|
" fig = plt.figure()\n",
|
|
" print(list(sortedScores[clfName][i].keys())[::revIndex])\n",
|
|
" print('#')\n",
|
|
" print(list(sortedScores[clfName][i].values())[::revIndex])\n",
|
|
" plt.barh(list(sortedScores[clfName][i].keys())[::revIndex], list(sortedScores[clfName][i].values())[::revIndex])\n",
|
|
" plt.title(label + ': ' + clfName + ' - ' + evaluationDatasetsTitle[i])\n",
|
|
" # plt.close(fig)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotTransformationScores(scores[0], scores[0], scores[4], 'TestScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotTransformationScores(scores[1], scores[0], scores[4], 'TrainScore', True)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotTransformationScores(scores[2], scores[0], scores[4], 'FitTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotTransformationScores(scores[3], scores[0], scores[4], 'ScoreTimes', False)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Test SVM with rbf kernel on datasets with split boat classes"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from sklearn.svm import SVC\n",
|
|
"svmClassifierWithParamsGrid = [\n",
|
|
" (SVC(), [\n",
|
|
" {'C': [0.1, 1, 10, 100]} \n",
|
|
" ])\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"svmScores = testClassifiers(svmClassifierWithParamsGrid, 10000)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Verify that SVM cannot be used for a datasets composed of more than 10.000 elements"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"svmClassifierWithParamsGridForLargeDatasets = [\n",
|
|
" (SVC(), [\n",
|
|
" {'C': [10]} \n",
|
|
" ])\n",
|
|
"]\n",
|
|
"svmScoresOnLargeDatasets = testClassifiers(svmClassifierWithParamsGridForLargeDatasets)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Conclusion:\n",
|
|
" \n",
|
|
"In terms of test score and scoring time, best single model (without boat class splitting) classifiers are:\n",
|
|
" * MLPClassifier with scaled inputs without PCA\n",
|
|
" * RandomForestClassifier without any scaling and PCA\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {
|
|
"ExecuteTime": {
|
|
"end_time": "2018-09-03T15:02:32.462718Z",
|
|
"start_time": "2018-09-03T15:02:32.459579Z"
|
|
}
|
|
},
|
|
"source": [
|
|
"## MLP vs. RandomForest"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Compare two MLP vs. RandomForest on polars dataset considering its optimal setup"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"(trainDatasetsInput, trainDatasetsOutput, testDatasetsInput, testDatasetsOutput) = splitTrainTest(evaluationDatasets)\n",
|
|
"trainDatasetsInput = trainDatasetsInput[0:4]\n",
|
|
"trainDatasetsInput.append(trainDatasetsInput[3].drop(['scaledSpeedBefore', 'scaledSpeedAfter', 'speedLossRatio', 'speedGainRatio', 'maxTurningRate', 'nextMarkAfter', 'nextMarkBefore'], axis=1))\n",
|
|
"trainDatasetsOutput = trainDatasetsOutput[0:4]\n",
|
|
"trainDatasetsOutput.append(trainDatasetsOutput[3])\n",
|
|
"trainDatasetsInput = trainDatasetsInput[4:5]\n",
|
|
"trainDatasetsOutput = trainDatasetsOutput[4:5]\n",
|
|
"scaledTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" scaledTrainDatasetsInput.append(StandardScaler().fit_transform(inputDataset))\n",
|
|
"transformedTrainDatasetsInput = []\n",
|
|
"for inputDataset in trainDatasetsInput:\n",
|
|
" transformedTrainDatasetsInput.append(transformInput(inputDataset))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"SMILE Library for java does not support ReLU activation function in its neural network implementation. Thats why we test only MLP with logistic sigmoid."
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"bestClassifiersWithParamsGrid = [\n",
|
|
" (RandomForestClassifier(), trainDatasetsInput[0], trainDatasetsOutput[0], [\n",
|
|
" {'n_estimators': [30, 60, 100, 500], 'max_depth': [None, 3, 5, 10]}\n",
|
|
" ]),\n",
|
|
" (MLPClassifier(), scaledTrainDatasetsInput[0], trainDatasetsOutput[0], [\n",
|
|
" {'hidden_layer_sizes': [(100), (200), (100, 100), (200, 200), (100, 100, 100), (200, 200, 200)], 'activation': ['logistic']}\n",
|
|
" ])\n",
|
|
"]"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def finetuneClassifiers(classifiersWithParamsGrid, dropFeatures=None):\n",
|
|
" testScores = []\n",
|
|
" trainScores = []\n",
|
|
" fitTimes = []\n",
|
|
" scoreTimes = []\n",
|
|
" clfNames = []\n",
|
|
" bestParams = []\n",
|
|
" for clfWithParamsGrid in classifiersWithParamsGrid:\n",
|
|
" clf = clfWithParamsGrid[0]\n",
|
|
" inputFeatures = clfWithParamsGrid[1]\n",
|
|
" outputFeatures = clfWithParamsGrid[2]\n",
|
|
" paramsGrid = clfWithParamsGrid[3]\n",
|
|
" clfName = clf.__class__.__name__\n",
|
|
" clfNames.append(clfName)\n",
|
|
" print('########################')\n",
|
|
" print('Classifier: ' + clfName)\n",
|
|
" gridSearch = determineBestScoreAndParams(inputFeatures, outputFeatures, clf, paramsGrid)\n",
|
|
" scores = gridSearch.cv_results_\n",
|
|
" testScore = max(scores['mean_test_score'])\n",
|
|
" j = np.argmax(scores['mean_test_score'])\n",
|
|
" trainScore = scores['mean_train_score'][j]\n",
|
|
" fitTime = scores['mean_fit_time'][j]\n",
|
|
" scoreTime = scores['mean_score_time'][j]\n",
|
|
" testScores.append(scores['mean_test_score'])\n",
|
|
" trainScores.append(scores['mean_train_score'])\n",
|
|
" fitTimes.append(scores['mean_fit_time'])\n",
|
|
" scoreTimes.append(scores['mean_score_time'])\n",
|
|
" bestParams = gridSearch.best_params_\n",
|
|
" print(\"testScore:\\t\" + str(testScore))\n",
|
|
" print(\"trainScore:\\t\" + str(trainScore))\n",
|
|
" print(\"fitTime:\\t\" + str(fitTime))\n",
|
|
" print(\"scoreTime:\\t\" + str(scoreTime))\n",
|
|
" print(\"best_params_:\\n\" + str(gridSearch.best_params_))\n",
|
|
" print(\"mean_test_scores:\" + str(scores['mean_test_score']))\n",
|
|
" print(\"mean_train_scores:\" + str(scores['mean_train_score']))\n",
|
|
" print(\"mean_fit_times:\" + str(scores['mean_fit_time']))\n",
|
|
" print(\"mean_score_times:\" + str(scores['mean_score_time']))\n",
|
|
" return (testScores, trainScores, fitTimes, scoreTimes, clfNames, bestParams)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"scoresTopClassifiers = finetuneClassifiers(bestClassifiersWithParamsGrid)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotFinetuningScores(scores):\n",
|
|
" for i in range(len(scores[0])):\n",
|
|
" testScores = scores[0][i]\n",
|
|
" trainScores = scores[1][i]\n",
|
|
" fitTimes = scores[2][i]\n",
|
|
" scoreTimes = scores[3][i]\n",
|
|
" clfName = scores[4][i]\n",
|
|
" fig = plt.figure()\n",
|
|
" plt.plot(testScores, trainScores) \n",
|
|
" plt.legend(['Test', 'Train'])\n",
|
|
" plt.title(clfName + ' - Test/Train')\n",
|
|
" fig = plt.figure()\n",
|
|
" plt.plot(fitTimes) \n",
|
|
" plt.title(clfName + ' - Fit time')\n",
|
|
" fig = plt.figure()\n",
|
|
" plt.plot(scoreTimes) \n",
|
|
" plt.title(clfName + ' - Score time')"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"plotFinetuningScores(scoresTopClassifiers)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"## Final classifier comparison"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Prepare test datasets"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"trainDatasetsInput[0].count()"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"len(testDatasetsInput)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"testDatasetsInput = testDatasetsInput[0:4]\n",
|
|
"testDatasetsInput.append(testDatasetsInput[3].drop(['scaledSpeedBefore', 'scaledSpeedAfter', 'speedLossRatio', 'speedGainRatio', 'maxTurningRate', 'nextMarkAfter', 'nextMarkBefore'], axis=1))\n",
|
|
"testDatasetsOutput = testDatasetsOutput[0:4]\n",
|
|
"testDatasetsOutput.append(testDatasetsOutput[3])\n",
|
|
"testDatasetsInput = testDatasetsInput[4:5]\n",
|
|
"testDatasetsOutput = testDatasetsOutput[4:5]\n",
|
|
"transformedTestDatasetsInput = []\n",
|
|
"for inputDataset in testDatasetsInput:\n",
|
|
" transformedTestDatasetsInput.append(transformInput(inputDataset))\n",
|
|
"scaledTestDatasetsInput = []\n",
|
|
"for inputDataset in testDatasetsInput:\n",
|
|
" scaledTestDatasetsInput.append(StandardScaler().fit_transform(inputDataset))"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Prepare classifiers with its best hyperparameters and appropriate dataset"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"clfsWithConfig = [\n",
|
|
" (LinearSVC(dual=False, C=1), 1)\n",
|
|
"]\n",
|
|
"preprocessingTrainInputOptions = {\n",
|
|
" 0 : trainDatasetsInput,\n",
|
|
" 1 : scaledTrainDatasetsInput,\n",
|
|
" 2 : transformedTrainDatasetsInput\n",
|
|
"}\n",
|
|
"preprocessingTestInputOptions = {\n",
|
|
" 0 : testDatasetsInput,\n",
|
|
" 1 : scaledTestDatasetsInput,\n",
|
|
" 2 : transformedTestDatasetsInput\n",
|
|
"}"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Compare best classifiers"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"from collections import defaultdict\n",
|
|
"def report2dict(cr):\n",
|
|
" tmp = list()\n",
|
|
" for row in cr.split(\"\\n\"):\n",
|
|
" parsed_row = [x for x in row.split(\" \") if len(x) > 0]\n",
|
|
" if len(parsed_row) > 0:\n",
|
|
" tmp.append(parsed_row)\n",
|
|
" measures = tmp[0]\n",
|
|
" D_class_data = defaultdict(dict)\n",
|
|
" for row in tmp[1:]:\n",
|
|
" class_label = row[0]\n",
|
|
" for j, m in enumerate(measures):\n",
|
|
" D_class_data[class_label][m.strip()] = float(row[j + 1].strip())\n",
|
|
" return D_class_data\n",
|
|
"def classificationReport(clf, inputFeatures, target):\n",
|
|
" clfName = clf.__class__.__name__\n",
|
|
" predicted = clf.predict(inputFeatures)\n",
|
|
" clfReport = classification_report(target, predicted)\n",
|
|
" print('##################')\n",
|
|
" print(clfName)\n",
|
|
" print(clfReport)\n",
|
|
" return report2dict(clfReport)"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for i in range(len(evaluationDatasetsTitle)):\n",
|
|
" print(evaluationDatasetsTitle[i])"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"finalComparison = {}\n",
|
|
"for clfWithConfig in clfsWithConfig:\n",
|
|
" i = 0\n",
|
|
" for testInput in testDatasetsInput:\n",
|
|
" x = preprocessingTrainInputOptions[clfWithConfig[1]][i]\n",
|
|
" y = trainDatasetsOutput[i]\n",
|
|
" clf = clfWithConfig[0]\n",
|
|
" print(evaluationDatasetsTitle[i])\n",
|
|
" print(x.shape)\n",
|
|
" print(preprocessingTestInputOptions[clfWithConfig[1]][i].shape)\n",
|
|
" clf.fit(x, y)\n",
|
|
" clfName = clf.__class__.__name__\n",
|
|
" x = preprocessingTestInputOptions[clfWithConfig[1]][i]\n",
|
|
" y = testDatasetsOutput[i]\n",
|
|
" finalComparison[clfName] = classificationReport(clf, x, y)\n",
|
|
" i += 1"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "markdown",
|
|
"metadata": {},
|
|
"source": [
|
|
"Plot comparison statistics"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def plotTestScore(scoreMetricName, target):\n",
|
|
" dataToPlot = []\n",
|
|
" dataset = [clazzMetrics[scoreMetricName] for clazzMetrics in finalComparison[target].values()]\n",
|
|
" tempScores = {}\n",
|
|
" i = 0\n",
|
|
" for clfName, clfMetrics in finalComparison.items():\n",
|
|
" tempScores[clfName] = clfMetrics[target][scoreMetricName]\n",
|
|
" i += 1\n",
|
|
" i = 1\n",
|
|
" sortedScores = OrderedDict()\n",
|
|
" for key, value in sorted(tempScores.items(), reverse=True, key=lambda kv: kv[1]):\n",
|
|
" print(str(i) + '. ' + key + \":\\t\" + str(value))\n",
|
|
" sortedScores[key] = value\n",
|
|
" i += 1\n",
|
|
" plt.figure()\n",
|
|
" plt.barh(list(sortedScores.keys())[::-1], list(sortedScores.values())[::-1])\n",
|
|
" plt.title(scoreMetricName + ' - ' + target) \n",
|
|
"# defaultdict(dict,\n",
|
|
"# {'avg / total': {'f1-score': 0.61,\n",
|
|
"# 'precision': 0.7,\n",
|
|
"# 'recall': 0.6,\n",
|
|
"# 'support': 5.0},\n",
|
|
"# 'class 0': {'f1-score': 0.67,\n",
|
|
"# 'precision': 0.5,\n",
|
|
"# 'recall': 1.0,\n",
|
|
"# 'support': 1.0},\n",
|
|
"# 'class 1': {'f1-score': 0.0,\n",
|
|
"# 'precision': 0.0,\n",
|
|
"# 'recall': 0.0,\n",
|
|
"# 'support': 1.0},\n",
|
|
"# 'class 2': {'f1-score': 0.8,\n",
|
|
"# 'precision': 1.0,\n",
|
|
"# 'recall': 0.67,\n",
|
|
"# 'support': 3.0}})"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": null,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"for scoreMetricName in ['f1-score', 'precision', 'recall']:\n",
|
|
" for target in finalComparison.values()[0].values().keys():\n",
|
|
" plotTestScore(scoreMetricName, target)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "Python 3",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.6.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|