dnrops
/
OpenGlass
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Small tweaks

This commit is contained in:
Franci Penov 2024-06-05 19:31:26 -07:00
parent 31fc374a10
commit cad29a1772
3 changed files with 29 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
sources/agent/imageDescription.ts
View File

@ -3,9 +3,9 @@ import { groqRequest } from "../modules/groq-llama3";
import { gptRequest } from "../modules/openai";
export async function imageDescription(src: Uint8Array, model?: KnownModel): Promise<string> {
export async function imageDescription(src: Uint8Array, model: KnownModel = 'moondream:1.8b-v2-fp16'): Promise<string> {
return ollamaInference({
model: model || 'moondream:1.8b-v2-moondream2-text-model-f16',
model: model,
messages: [{
role: 'system',
content: 'You are a very advanced model and your task is to describe the image as precisely as possible. Transcribe any text you see.'

2
sources/modules/ollama.ts
View File

@ -37,5 +37,5 @@ export async function ollamaInference(args: {
});
return resp.data;
});
return trimIdent((response.message.content as string));
return trimIdent(((response.message?.content ?? '') as string));
}

15
sources/modules/openai.ts
View File

@ -7,6 +7,11 @@ export async function transcribeAudio(audioPath: string) {
try {
const response = await axios.post("https://api.openai.com/v1/audio/transcriptions", {
audio: audioBase64,
}, {
headers: {
'Authorization': `Bearer ${keys.openai}`, // Replace YOUR_API_KEY with your actual OpenAI API key
'Content-Type': 'application/json'
},
});
return response.data;
} catch (error) {
@ -63,6 +68,11 @@ export async function describeImage(imagePath: string) {
try {
const response = await axios.post("https://api.openai.com/v1/images/descriptions", {
image: imageBase64,
}, {
headers: {
'Authorization': `Bearer ${keys.openai}`, // Replace YOUR_API_KEY with your actual OpenAI API key
'Content-Type': 'application/json'
},
});
return response.data;
} catch (error) {
@ -79,6 +89,11 @@ export async function gptRequest(systemPrompt: string, userPrompt: string) {
{ role: "system", content: systemPrompt },
{ role: "user", content: userPrompt },
],
}, {
headers: {
'Authorization': `Bearer ${keys.openai}`, // Replace YOUR_API_KEY with your actual OpenAI API key
'Content-Type': 'application/json'
},
});
return response.data;
} catch (error) {