Skip to content

Commit

Permalink
Merge pull request #140 from CogStack/small-metrics-improvements
Browse files Browse the repository at this point in the history 
Further metrics view improvements
  • Loading branch information
@tomolopolis
tomolopolis authored Jul 17, 2023
2 parents 65196a3 + bd63f20 commit 0687acd
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 24 deletions.
13 changes: 12 additions & 1 deletion webapp/api/api/views.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -636,7 +636,18 @@ def model_loaded(_):
def metrics(request):
p_ids = request.GET.get('projectIds').split(',')
projects = ProjectAnnotateEntities.objects.filter(id__in=p_ids)
# assume projects all use the same model for eval purposes.

# provide warning of inconsistent models used or for models that are not loaded.
p_cdbs = set(p.concept_db for p in projects)
if len(p_cdbs) > 1:
logger.warning('Inconsistent CDBs used in the generation of metrics - should use the same CDB for '
f'consistent results - found {[cdb.name for cdb in p_cdbs]} - metrics will only use the first'
f' CDB {projects[0].concept_db.name}')
for p_cdb in p_cdbs:
if p_cdb not in CDB_MAP:
logger.warning(f'CDB {p_cdb.name} not in CDB_MAP cache - this will now be loaded - '
f'and will not show intermediary training status')

cat = get_medcat(CDB_MAP=CDB_MAP, VOCAB_MAP=VOCAB_MAP,
CAT_MAP=CAT_MAP, project=projects[0])
project_data = retrieve_project_data(projects)
Expand Down
1 change: 0 additions & 1 deletion webapp/frontend/src/App.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,6 @@ export default {
}
.link {
//padding-top: 10px;
display:inline-block;
height: 25px;
cursor: pointer;
Expand Down
22 changes: 19 additions & 3 deletions webapp/frontend/src/components/anns/AnnoResult.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
<td class="doc-name">{{result['document name']}}</td>
<td class="cui">{{result.cui}}</td>
<td class="source-value">{{result['source value']}}</td>
<td class="accu">{{result.acc}}</td>
<td class="accu">{{Number(result.acc).toFixed(3)}}</td>
<td class="text">
<v-runtime-template :template="text"></v-runtime-template>
</td>
Expand Down Expand Up @@ -33,7 +33,6 @@ export default {
if (this.type === 'fp' || this.type === 'fn') {
highlightClass = 'highlight-task-1'
}
const srcVal = this.result['source value']
const resTxt = this.result.text
const regexp = RegExp(`${srcVal}`, 'sg')
Expand All @@ -45,13 +44,30 @@ export default {
} else {
outText += `${resTxt.slice(matches[matches.indexOf(match) - 1].index + srcVal.length, match.index)}`
}
outText += `<span class="${highlightClass}">${srcVal}</span>`
outText += `<span class="${highlightClass}" @click="openAnno">${srcVal}</span>`
if (matches.length === 1 || match === matches[-1]) {
outText += `${resTxt.slice(match.index + srcVal.length)}</span>`
}
}
return outText
}
},
methods: {
openAnno () {
const routeData = this.$router.resolve(
{
name: 'train-annotations',
params: {
projectId: this.result['project id'],
docId: this.result['document id'],
},
query: {
annoStart: this.result['start'],
annoEnd: this.result['end']
}
})
window.open(routeData.href, '_blank');
}
}
}
</script>
Expand Down
9 changes: 6 additions & 3 deletions webapp/frontend/src/views/Home.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,14 @@
v-if="!loadingProjects"
@row-selected="select">
<template #head(metrics)="data">
Metrics
<div id="metrics-head">Metrics</div>
<button class="btn btn-outline-primary load-metrics" @click="loadMetrics" v-if="selectedProjects.length > 0">
<font-awesome-icon icon="chevron-right"></font-awesome-icon>
</button>
<b-tooltip target="metrics-head"
triggers="hover"
container="projectTable"
title="Access the metrics view for a single or group of projects"></b-tooltip>
</template>
<template #head(cuis)="">
<div id="cuis-header">Concepts</div>
Expand Down Expand Up @@ -119,7 +123,7 @@
<transition name="alert"><div class="alert alert-danger" v-if="modelSavedError" role="alert">Error saving model</div></transition>
<transition name="alert"><div class="alert alert-primary" v-if="loadingModel" role="alert">Loading model</div></transition>
<transition name="alert"><div class="alert alert-danger" v-if="modelCacheLoadError" role="alert">Error loading MedCAT model for project</div></transition>
<transition name="alert"><div class="alert alert-danger" v-if="projectLockedWarning" role="alert">Unable load a locked project. Unlock via /admin/</div></transition>
<transition name="alert"><div class="alert alert-danger" v-if="projectLockedWarning" role="alert">Unable load a locked project. Contact your CogStack administrator to unlock</div></transition>
</div>
<modal v-if="clearModelModal" :closable="true" @modal:close="clearModelModal = false">
<div slot="header">
Expand Down Expand Up @@ -174,7 +178,6 @@ export default {
'anno_class',
'cdb_search_filter',
'model_loaded',
'metrics',
'save_model'
]
},
Expand Down
145 changes: 129 additions & 16 deletions webapp/frontend/src/views/Metrics.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,94 @@
<b-table striped hover small :items="annoSummary.items" :fields="annoSummary.fields"></b-table>
</b-tab>
<b-tab title="Concept Summary" class="concept-summary">
<b-table striped hover small :items="conceptSummary.items" :fields="conceptSummary.fields">
<b-table striped hover small :items="conceptSummary.items" :fields="conceptSummary.fields" id="concepts-sum-tbl">
<template #head(concept)="data">
<div id="concept-head">Concept</div>
<b-tooltip target="concept-head"
triggers="hover"
container="concepts-sum-tbl"
title=""></b-tooltip>
</template>
<template #head(concept_count)="data">
<div id="concept-count-head">Concept Count</div>
<b-tooltip target="concept-count-head"
triggers="hover"
container="concepts-sum-tbl"
title="Number of occurrences across the projects"></b-tooltip>
</template>
<template #head(variations)="data">
<div id="variations-head"># Vars</div>
<b-tooltip target="variations-head"
triggers="hover"
container="concepts-sum-tbl"
title="The count of unique variations for a concept"></b-tooltip>
</template>
<template #head(variation_values)="data">
<div id="variations-values-head">Variations</div>
<b-tooltip target="variations-values-head"
triggers="hover"
container="concepts-sum-tbl"
title="The unique set of variations for a concept"></b-tooltip>
</template>
<template #head(count_variations_ratio)="data">
<div id="variations-ratio-head">Variations Ratio</div>
<b-tooltip target="variations-ratio-head"
triggers="hover"
container="concepts-sum-tbl"
title="The ratio of number of annotations and the number of variations of a concept"></b-tooltip>
</template>
<template #head(cui)="data">
<div id="cui-head">CUI</div>
<b-tooltip target="cui-head"
triggers="hover"
container="concepts-sum-tbl"
title="The Concept Unique Identifier"></b-tooltip>
</template>
<template #head(cui_f1)="data">
<div id="cui-f1-head">F1</div>
<b-tooltip target="cui-f1-head"
triggers="hover"
container="concepts-sum-tbl"
title="The harmonic mean of the recall and precision scores"></b-tooltip>
</template>
<template #head(cui_prec)="data">
<div id="cui-prec-head">Prec</div>
<b-tooltip target="cui-prec-head"
triggers="hover"
container="concepts-sum-tbl"
title="The precision scores of a concept."></b-tooltip>
</template>
<template #head(cui_rec)="data">
<div id="cui-rec-head">Rec</div>
<b-tooltip target="cui-rec-head"
triggers="hover"
container="concepts-sum-tbl"
title="The recall scores of a concept."></b-tooltip>
</template>
<template #head(tps)="data">
<div id="tps-head">TPs</div>
<b-tooltip target="tps-head"
triggers="hover"
container="concepts-sum-tbl"
title="True positives - concept examples that are annotated and predicted by the model"></b-tooltip>
</template>
<template #head(fns)="data">
<div id="fns-head">FNs</div>
<b-tooltip target="fns-head"
triggers="hover"
container="concepts-sum-tbl"
title="False negatives - concept examples that annotated but not predicted by the model"></b-tooltip>
</template>
<template #head(fps)="data">
<div id="fps-head">FPs</div>
<b-tooltip target="fps-head"
triggers="hover"
container="concepts-sum-tbl"
title="False positives - concept examples that are predicted but not annotated"></b-tooltip>
</template>
<template #cell(variation_values)="data">
<div>{{data.item.value.join(', ')}}</div>
</template>
<template #cell(cui_f1)="data">
<div v-html="data.value"></div>
</template>
Expand All @@ -30,20 +117,17 @@
@click="openExamples('tp_examples', data.item)">
{{data.item.tps}}
</button>
<!-- <span v-if="data.item.tps === 0">{{data.item.tps}}</span>-->
</template>
<template #cell(fns)="data">
<button class="btn btn-outline-warning res-btn" :disabled="data.item.fns === 0"
@click="openExamples('fn_examples', data.item)">
{{data.item.fns}}
</button>
<!-- <span v-if="data.item.fns === 0">{{data.item.fns}}</span>-->
</template>
<template #cell(fps)="data">
<button class="btn btn-outline-danger res-btn" :disabled="data.item.fps === 0" @click="openExamples('fp_examples', data.item)">
{{data.item.fps}}
</button>
<!-- <span v-if="data.item.fps === 0">{{data.item.fps}}</span>-->
</template>
</b-table>
</b-tab>
Expand All @@ -52,9 +136,24 @@
</b-tab>
</b-tabs>
</div>
<modal class="summary-modal" v-if="predictedResults" :closable="true" @modal:close="predictedResults = null">
<h3 slot="header">{{predictionResultsTitle}}</h3>
<modal class="summary-modal" v-if="modalData.results" :closable="true" @modal:close="clearModalData">
<h3 slot="header">{{modalData.title}}</h3>
<div slot="body">
<div v-if="modalData.type === 'fp'">
<p>False positive model predictions can be the result of:</p>
<ul>
<li>Alternative model predictions that are overlapping with other concepts</li>
<li>Genuine missed annotations by an annotator.</li>
</ul>
<p>Clicking through these annotations will not highlight this annotation as it doesn't exist in the dataset </p>
</div>
<div v-if="modalData.type === 'fn'">
<p>False negative model predictions can be the result of:</p>
<ul>
<li>An model mistake that marked an annotation 'correct' where it should be incorrect</li>
<li>An annotator mistake that marked an annotation 'correct' where it should be incorrect</li>
</ul>
</div>
<table class="table table-hover">
<thead>
<tr>
Expand All @@ -66,7 +165,7 @@
</tr>
</thead>
<tbody>
<anno-result v-for="(res, key) of predictedResults" :key="key" :result="res"></anno-result>
<anno-result v-for="(res, key) of modalData.results" :key="key" :result="res" :type="modalData.type"></anno-result>
</tbody>
</table>
</div>
Expand Down Expand Up @@ -121,8 +220,10 @@ export default {
},
conceptSummary: {
fields: [
{ key: 'concept_count', label: 'Count', sortable: true },
{ key: 'concept_name', label: 'Concept', sortable: true },
{ key: 'concept_name', sortable: true },
{ key: 'concept_count', sortable: true },
{ key: 'variations', sortable: true },
{ key: 'variation_values', label: ''},
{ key: 'count_variations_ratio', label: 'Variation Ratio', sortable: true },
{ key: 'cui', label: 'CUI' },
{ key: 'cui_f1', label: 'F1', sortable: true, formatter: this.perfFormatter },
Expand All @@ -136,21 +237,34 @@ export default {
metaAnnsSummary: {
fields: []
},
predictedResults: null,
predictionResultsTitle: null
modalData: {
results: null,
title: null,
type: null
}
}
},
methods: {
clearModalData () {
this.modalData = {
results: null,
title: null,
type: null
}
},
openExamples (exampleType, item) {
if (exampleType === 'tp_examples') {
this.predictionResultsTitle = 'True Positive Model Predictions'
this.modalData.title = 'True Positive Model Predictions'
this.modalData.type = 'tp'
} else if (exampleType === 'fp_examples') {
this.predictionResultsTitle = 'False Positive Model Predictions'
this.modalData.title = 'False Positive Model Predictions'
this.modalData.type = 'fp'
} else {
this.predictionResultsTitle = 'False Negative Model Predictions'
this.modalData.title = 'False Negative Model Predictions'
this.modalData.type = 'fn'
}
const idx = this.conceptSummary.items.indexOf(item)
this.predictedResults = this.conceptSummary.items[idx][exampleType]
this.modalData.results = this.conceptSummary.items[idx][exampleType]
},
perfFormatter (value) {
let txtColorClass = 'good-perf'
Expand Down Expand Up @@ -209,7 +323,6 @@ $metrics-header-height: 42px;
.concept-summary {
overflow-y: auto;
height: 100%
}
.meta-anno-summary {
Expand Down
9 changes: 9 additions & 0 deletions webapp/frontend/src/views/TrainAnnotations.vue
View file
Original file line number Diff line number Diff line change
Expand Up @@ -485,6 +485,15 @@ export default {
this.metaAnnotate = this.currentEnt && (this.currentEnt.assignedValues[TASK_NAME] === CONCEPT_ALTERNATIVE ||
this.currentEnt.assignedValues[TASK_NAME] === CONCEPT_CORRECT)
this.loadingDoc = false
if (this.$route.query.annoStart && this.$route.query.annoEnd) {
const ent = _.find(this.ents, e => {
return Number(this.$route.query.annoStart) === e.start_ind &
Number(this.$route.query.annoEnd) === e.end_ind
})
if (ent) {
this.currentEnt = ent
}
}
}
})
},
Expand Down

0 comments on commit 0687acd

Please sign in to comment.