enhance illustration being taken into account in the response

This commit is contained in:
Adrien
2026-04-12 16:26:25 +02:00
parent 820734c251
commit 767d1e2dbc
6 changed files with 272 additions and 30 deletions
@@ -29,7 +29,7 @@ public class ChatService {
- Use clear structure: headings, bullet points, or numbered steps where appropriate to maximize clarity - Use clear structure: headings, bullet points, or numbered steps where appropriate to maximize clarity
- Only say you cannot answer if the context is entirely unrelated to the question - Only say you cannot answer if the context is entirely unrelated to the question
- Cite sources for each major claim using the reference labels from the context (e.g. [S1], [F2]). Prefer these labels over inventing page numbers, but you may also describe the source naturally if needed. - Cite sources for each major claim using the reference labels from the context (e.g. [S1], [F2]). Prefer these labels over inventing page numbers, but you may also describe the source naturally if needed.
- When referencing diagrams or figures, prefer their label from the context (e.g. [F1]) - Figures (labeled [F1], [F2], etc.) are actual images and drawings from the textbook — they will be rendered as inline illustrations in your response. Use them actively to support your explanations: reference a figure when it visually demonstrates anatomy, a surgical step, or a clinical concept you are describing.
- Maintain continuity with the conversation history - Maintain continuity with the conversation history
- Never fabricate clinical information not present in the context - Never fabricate clinical information not present in the context
"""; """;
@@ -14,9 +14,15 @@ public record TopicSummaryResponse(
Instant generatedAt Instant generatedAt
) { ) {
public record SourceReference( public record SourceReference(
String type,
String refLabel,
String bookId, String bookId,
String bookTitle, String bookTitle,
Integer page Integer page,
String figureId,
String label,
String caption,
String imageUrl
) { ) {
} }
} }
@@ -35,6 +35,7 @@ public class TopicSummaryService {
- Structure your response clearly with key points - Structure your response clearly with key points
- Cite claims using ONLY the reference labels provided in the context (e.g. [S1], [F2]). - Cite claims using ONLY the reference labels provided in the context (e.g. [S1], [F2]).
Do not invent page numbers, section titles, or labels not present in the CONTEXT block. Do not invent page numbers, section titles, or labels not present in the CONTEXT block.
- Figures (labeled [F1], [F2], etc.) are actual images and drawings from the textbook — they will be rendered as inline illustrations in your response. Use them actively to support your explanations: reference a figure when it visually demonstrates anatomy, a surgical step, or a clinical concept you are describing.
- If the retrieved context does not contain sufficient information on the topic, - If the retrieved context does not contain sufficient information on the topic,
explicitly state: "The uploaded books do not contain sufficient information on this topic." explicitly state: "The uploaded books do not contain sufficient information on this topic."
- Never hallucinate or fabricate clinical information - Never hallucinate or fabricate clinical information
@@ -135,7 +136,7 @@ public class TopicSummaryService {
return String.format( return String.format(
"Provide a comprehensive educational summary of the following neurosurgery topic: " + "Provide a comprehensive educational summary of the following neurosurgery topic: " +
"%s. Topic description: %s. " + "%s. Topic description: %s. " +
"Include key concepts, clinical considerations, and important details that a neurosurgeon should know.", "Include key concepts, diagrams, illustations and clinical considerations, and important details that a neurosurgeon should know.",
topic.getName(), topic.getDescription() topic.getName(), topic.getDescription()
); );
} }
@@ -165,7 +166,7 @@ public class TopicSummaryService {
.append(f.getCaption() != null ? f.getCaption() : "") .append(f.getCaption() != null ? f.getCaption() : "")
.append("\n"); .append("\n");
} }
sb.append("\n"); sb.append("\nWhen referencing diagrams, use their label from the context (e.g. [F1]).\n\n");
} }
sb.append("QUESTION:\n").append(question); sb.append("QUESTION:\n").append(question);
@@ -177,27 +178,35 @@ public class TopicSummaryService {
List<Book> readyBooks) { List<Book> readyBooks) {
List<TopicSummaryResponse.SourceReference> sources = new ArrayList<>(); List<TopicSummaryResponse.SourceReference> sources = new ArrayList<>();
for (SectionEntity s : sections) { for (int i = 0; i < sections.size(); i++) {
SectionEntity s = sections.get(i);
Book book = readyBooks.stream() Book book = readyBooks.stream()
.filter(b -> b.getId().equals(s.getBookId())) .filter(b -> b.getId().equals(s.getBookId()))
.findFirst() .findFirst()
.orElse(null); .orElse(null);
String title = book != null ? book.getTitle() : "Book"; String title = book != null ? book.getTitle() : "Book";
String bookId = book != null ? book.getId().toString() : null; String bookId = book != null ? book.getId().toString() : null;
sources.add(new TopicSummaryResponse.SourceReference(bookId, title, s.getPageStart())); sources.add(new TopicSummaryResponse.SourceReference(
"TEXT", "S" + (i + 1), bookId, title, s.getPageStart(),
null, null, null, null));
} }
for (FigureEntity f : figures) { for (int i = 0; i < figures.size(); i++) {
FigureEntity f = figures.get(i);
Book book = readyBooks.stream() Book book = readyBooks.stream()
.filter(b -> b.getId().equals(f.getBookId())) .filter(b -> b.getId().equals(f.getBookId()))
.findFirst() .findFirst()
.orElse(null); .orElse(null);
String title = book != null ? book.getTitle() : "Book"; String title = book != null ? book.getTitle() : "Book";
String bookId = book != null ? book.getId().toString() : null; String bookId = book != null ? book.getId().toString() : null;
sources.add(new TopicSummaryResponse.SourceReference(bookId, title, f.getPage())); String filename = f.getImagePath().substring(f.getImagePath().lastIndexOf('/') + 1);
String imageUrl = "/api/v1/figures/" + f.getBookId() + "/" + filename;
sources.add(new TopicSummaryResponse.SourceReference(
"FIGURE", "F" + (i + 1), bookId, title, f.getPage(),
f.getId(), f.getLabel(), f.getCaption(), imageUrl));
} }
return sources.stream().distinct().toList(); return sources;
} }
private String serializeSources(List<TopicSummaryResponse.SourceReference> sources) { private String serializeSources(List<TopicSummaryResponse.SourceReference> sources) {
+52 -3
View File
@@ -87,12 +87,37 @@ const isUser = computed(() => props.message.role === 'USER')
const activeRef = ref<string | null>(null) const activeRef = ref<string | null>(null)
const sourceListEl = ref<HTMLElement | null>(null) const sourceListEl = ref<HTMLElement | null>(null)
/** Replaces [S1]/[F1]-style labels in the rendered HTML with clickable badges. */ function escapeHtml(s: string): string {
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
}
/** Replaces [S1]/[F1]-style labels in the rendered HTML with clickable badges.
* For figure citations, also injects an inline illustration below the badge. */
const renderedWithBadges = computed(() => { const renderedWithBadges = computed(() => {
const html = marked.parse(props.message.content) as string const html = marked.parse(props.message.content) as string
const figureMap = new Map<string, ChatSource>()
for (const src of (props.message.sources ?? [])) {
if (src.type === 'FIGURE' && src.refLabel) {
figureMap.set(src.refLabel, src)
}
}
return html.replace(/\[(S|F)\d+\]/g, (match) => { return html.replace(/\[(S|F)\d+\]/g, (match) => {
const inner = match.slice(1, -1) // e.g. "S1" const inner = match.slice(1, -1) // e.g. "F1"
return `<span class="citation-badge" data-ref="${inner}" title="Jump to source ${inner}">${match}</span>` const badge = `<span class="citation-badge" data-ref="${inner}" title="Jump to source ${inner}">${match}</span>`
const fig = figureMap.get(inner)
if (fig?.imageUrl) {
const alt = escapeHtml(fig.caption || fig.label || 'Figure')
const captionText = [fig.label, fig.caption].filter(Boolean).map(escapeHtml).join(' — ')
const captionHtml = captionText
? `<figcaption class="inline-figure-caption">${captionText}</figcaption>`
: ''
return `${badge}<figure class="inline-figure"><img src="${fig.imageUrl}" alt="${alt}" class="inline-figure-img" loading="lazy" onerror="this.parentElement.style.display='none'" />${captionHtml}</figure>`
}
return badge
}) })
}) })
@@ -426,6 +451,30 @@ function formatTime(iso: string): string {
color: #276749; color: #276749;
} }
.message-content--markdown :deep(.inline-figure) {
display: block;
margin: 0.75rem 0;
text-align: center;
}
.message-content--markdown :deep(.inline-figure-img) {
max-width: 100%;
max-height: 400px;
border-radius: 6px;
border: 1px solid #e2e8f0;
object-fit: contain;
display: block;
margin: 0 auto;
}
.message-content--markdown :deep(.inline-figure-caption) {
font-size: 0.78rem;
color: #718096;
font-style: italic;
margin-top: 0.3rem;
text-align: center;
}
.message-timestamp { .message-timestamp {
font-size: 0.7rem; font-size: 0.7rem;
opacity: 0.6; opacity: 0.6;
+6
View File
@@ -10,9 +10,15 @@ export interface Topic {
} }
export interface SourceReference { export interface SourceReference {
type?: 'TEXT' | 'FIGURE'
refLabel?: string
bookId: string | null bookId: string | null
bookTitle: string bookTitle: string
page: number | null page: number | null
figureId?: string
label?: string
caption?: string
imageUrl?: string
} }
export interface TopicSummary { export interface TopicSummary {
+184 -12
View File
@@ -97,20 +97,57 @@
<span>{{ showSources ? '▲' : '▼' }}</span> <span>{{ showSources ? '▲' : '▼' }}</span>
</button> </button>
<div v-if="showSources" class="sources-list"> <div v-if="showSources" class="sources-list">
<!-- TEXT sources -->
<div <div
v-for="(source, idx) in topicStore.activeSummary.sources" v-for="(source, idx) in textSources"
:key="idx" :key="'text-' + idx"
class="source-chip" class="source-item"
:data-ref-label="source.refLabel"
>
<div
class="source-chip source-chip--text"
:class="{ 'source-chip--clickable': source.bookId && source.page }" :class="{ 'source-chip--clickable': source.bookId && source.page }"
@click="source.bookId && source.page ? handleOpenSource(source.bookId, source.page) : undefined" @click="source.bookId && source.page ? handleOpenSource(source.bookId, source.page) : undefined"
> >
<span class="source-icon">📖</span> <span class="source-icon">📖</span>
<span v-if="source.refLabel" class="source-ref-label">{{ source.refLabel }}</span>
<span class="source-book">{{ source.bookTitle }}</span> <span class="source-book">{{ source.bookTitle }}</span>
<span v-if="source.page" class="source-page">p.&nbsp;{{ source.page }}</span> <span v-if="source.page" class="source-page">p.&nbsp;{{ source.page }}</span>
<span v-if="source.bookId && source.page" class="source-open-hint"></span> <span v-if="source.bookId && source.page" class="source-open-hint"></span>
</div> </div>
</div> </div>
<!-- FIGURE sources -->
<div
v-for="(source, idx) in figureSources"
:key="'fig-' + idx"
class="source-item source-item--figure"
:data-ref-label="source.refLabel"
>
<div
class="source-chip source-chip--figure"
:class="{ 'source-chip--clickable': source.bookId && source.page }"
@click="source.bookId && source.page ? handleOpenSource(source.bookId, source.page) : undefined"
>
<span class="source-icon">🖼</span>
<span v-if="source.refLabel" class="source-ref-label source-ref-label--figure">{{ source.refLabel }}</span>
<span class="source-figure-label">{{ source.label || 'Figure' }}</span>
<span v-if="source.page" class="source-page">p.&nbsp;{{ source.page }}</span>
<span v-if="source.bookId && source.page" class="source-open-hint"></span>
</div>
<div v-if="source.caption" class="source-caption">{{ source.caption }}</div>
<div v-if="source.imageUrl" class="source-figure-image">
<img
:src="source.imageUrl"
:alt="source.caption || source.label || 'Figure'"
class="figure-img"
loading="lazy"
@error="onImageError"
/>
</div>
</div>
</div>
<BookPagePanel <BookPagePanel
v-if="readerPanel" v-if="readerPanel"
:book-id="readerPanel.bookId" :book-id="readerPanel.bookId"
@@ -146,7 +183,7 @@
import { ref, computed, onMounted, inject } from 'vue' import { ref, computed, onMounted, inject } from 'vue'
import { marked } from 'marked' import { marked } from 'marked'
import { RouterLink } from 'vue-router' import { RouterLink } from 'vue-router'
import { useTopicStore, type SavedSummaryItem } from '@/stores/topicStore' import { useTopicStore, type SavedSummaryItem, type SourceReference } from '@/stores/topicStore'
import { useBookStore } from '@/stores/bookStore' import { useBookStore } from '@/stores/bookStore'
import TopicCard from '@/components/TopicCard.vue' import TopicCard from '@/components/TopicCard.vue'
import BookPagePanel from '@/components/BookPagePanel.vue' import BookPagePanel from '@/components/BookPagePanel.vue'
@@ -166,10 +203,55 @@ const readerPanel = ref<ReaderPanel | null>(null)
const summaryTopics = computed(() => topicStore.topics.filter(t => t.id !== 'free-form')) const summaryTopics = computed(() => topicStore.topics.filter(t => t.id !== 'free-form'))
const textSources = computed(() =>
(topicStore.activeSummary?.sources ?? []).filter(s => s.type === 'TEXT' || !s.type)
)
const figureSources = computed(() =>
(topicStore.activeSummary?.sources ?? []).filter(s => s.type === 'FIGURE')
)
function onImageError(e: Event) {
const img = e.target as HTMLImageElement
img.style.display = 'none'
const wrapper = img.parentElement
if (wrapper) {
wrapper.innerHTML = '<span class="figure-missing">Image unavailable</span>'
}
}
function escapeHtml(s: string): string {
return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;')
}
const renderedSummary = computed(() => { const renderedSummary = computed(() => {
if (!topicStore.activeSummary) return '' if (!topicStore.activeSummary) return ''
const html = marked.parse(topicStore.activeSummary.summary) as string const html = marked.parse(topicStore.activeSummary.summary) as string
return html.replace(/\[S(\d+)\]/g, '<span class="source-ref">[S$1]</span>')
const figureMap = new Map<string, SourceReference>()
for (const src of topicStore.activeSummary.sources) {
if (src.type === 'FIGURE' && src.refLabel) {
figureMap.set(src.refLabel, src)
}
}
return html.replace(/\[(S|F)\d+\]/g, (match) => {
const inner = match.slice(1, -1)
const badge = `<span class="source-ref" data-ref="${inner}" title="Jump to source ${inner}">${match}</span>`
const fig = figureMap.get(inner)
if (fig?.imageUrl) {
const alt = escapeHtml(fig.caption || fig.label || 'Figure')
const captionText = [fig.label, fig.caption].filter(Boolean).map(escapeHtml).join(' — ')
const captionHtml = captionText
? `<figcaption class="inline-figure-caption">${captionText}</figcaption>`
: ''
return `${badge}<figure class="inline-figure"><img src="${fig.imageUrl}" alt="${alt}" class="inline-figure-img" loading="lazy" onerror="this.parentElement.style.display='none'" />${captionHtml}</figure>`
}
return badge
})
}) })
function handleSummaryClick(e: MouseEvent) { function handleSummaryClick(e: MouseEvent) {
@@ -484,19 +566,37 @@ function formatDateShort(iso: string): string {
.sources-list { .sources-list {
display: flex; display: flex;
flex-wrap: wrap; flex-direction: column;
gap: 0.5rem; gap: 0.5rem;
} }
.source-chip { .source-item {
display: flex; display: flex;
align-items: center; flex-direction: column;
gap: 0.25rem;
}
.source-item--figure {
gap: 0.4rem; gap: 0.4rem;
}
.source-chip {
display: inline-flex;
align-items: center;
gap: 0.25rem;
border-radius: 4px;
padding: 0.2rem 0.5rem;
font-size: 0.78rem;
}
.source-chip--text {
background: #ebf8ff; background: #ebf8ff;
border: 1px solid #bee3f8; border: 1px solid #bee3f8;
border-radius: 6px; }
padding: 0.3rem 0.7rem;
font-size: 0.8rem; .source-chip--figure {
background: #f0fff4;
border: 1px solid #9ae6b4;
} }
.source-chip--clickable { .source-chip--clickable {
@@ -504,20 +604,44 @@ function formatDateShort(iso: string): string {
transition: background 0.15s, border-color 0.15s; transition: background 0.15s, border-color 0.15s;
} }
.source-chip--clickable:hover { .source-chip--text.source-chip--clickable:hover {
background: #bee3f8; background: #bee3f8;
border-color: #90cdf4; border-color: #90cdf4;
} }
.source-chip--figure.source-chip--clickable:hover {
background: #c6f6d5;
border-color: #68d391;
}
.source-icon { .source-icon {
font-size: 0.8rem; font-size: 0.8rem;
} }
.source-ref-label {
font-size: 0.72rem;
font-weight: 700;
background: #bee3f8;
color: #2b6cb0;
border-radius: 3px;
padding: 0 0.3rem;
}
.source-ref-label--figure {
background: #9ae6b4;
color: #276749;
}
.source-book { .source-book {
color: #2b6cb0; color: #2b6cb0;
font-weight: 500; font-weight: 500;
} }
.source-figure-label {
color: #276749;
font-weight: 600;
}
.source-page { .source-page {
color: #718096; color: #718096;
} }
@@ -528,6 +652,30 @@ function formatDateShort(iso: string): string {
margin-left: 0.1rem; margin-left: 0.1rem;
} }
.source-caption {
font-size: 0.78rem;
color: #4a5568;
font-style: italic;
}
.source-figure-image {
max-width: 100%;
}
.figure-img {
max-width: 100%;
max-height: 300px;
border-radius: 6px;
border: 1px solid #e2e8f0;
object-fit: contain;
}
.figure-missing {
font-size: 0.78rem;
color: #a0aec0;
font-style: italic;
}
.no-sources { .no-sources {
font-size: 0.85rem; font-size: 0.85rem;
color: #a0aec0; color: #a0aec0;
@@ -563,6 +711,30 @@ function formatDateShort(iso: string): string {
font-style: italic; font-style: italic;
} }
.summary-text--markdown :deep(.inline-figure) {
display: block;
margin: 0.75rem 0;
text-align: center;
}
.summary-text--markdown :deep(.inline-figure-img) {
max-width: 100%;
max-height: 400px;
border-radius: 6px;
border: 1px solid #e2e8f0;
object-fit: contain;
display: block;
margin: 0 auto;
}
.summary-text--markdown :deep(.inline-figure-caption) {
font-size: 0.78rem;
color: #718096;
font-style: italic;
margin-top: 0.3rem;
text-align: center;
}
.summary-text--markdown :deep(.source-ref) { .summary-text--markdown :deep(.source-ref) {
color: #3182ce; color: #3182ce;
font-weight: 600; font-weight: 600;