AIを使用した動画要約機能の開発

# installs nvm (Node Version Manager)
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash

# download and install Node.js
nvm install 16

# verifies the right Node.js version is in the environment
node -v # should print `v16.20.2`

# verifies the right NPM version is in the environment
npm -v # should print `8.19.4``

sudo apt update
sudo apt install ffmpeg

sudo systemctl status gridstore

sudo systemctl start gridstore

git clone https://github.com/junwatu/video-summarizer-nodejs-griddb.git

cd video-summarizer-nodejs-griddb
npm install

cp .env .example .env

OPENAI_API_KEY=sk-....
VITE_API_URL=http://localhost:3000

npm run start:build

export function extractFrames(videoPath, secondsPerFrame, outputFolder) {
    return new Promise((resolve, reject) => {
        const frameRate = 1 / secondsPerFrame
        const framePattern = path.join(outputFolder, 'frame-%03d.png')

        ffmpeg(videoPath)
            .outputOptions([`-vf fps=${frameRate}`])
            .output(framePattern)
            .on('end', () => {
                fs.readdir(outputFolder, (err, files) => {
                    if (err) {
                        reject(err)
                    } else {
                        const framePaths = files.map(file => path.join(outputFolder, file))
                        resolve(framePaths)
                    }
                })
            })
            .on('error', reject)
            .run()
    })
}

export function imageToBase64(imagePath) {
    return new Promise((resolve, reject) => {
        fs.readFile(imagePath, (err, data) => {
            if (err) {
                reject(err)
            } else {
                const base64String = data.toString('base64')
                resolve(base64String)
            }
        })
    })
}

// Function to extract audio from video
export function extractAudio(videoPath, audioPath) {
    return new Promise((resolve, reject) => {
        ffmpeg(videoPath)
            .output(audioPath)
            .audioBitrate('32k')
            .on('end', resolve)
            .on('error', reject)
            .run()
    })
}

async function transcribeAudio(filePath) {
    try {
        const transcription = await openai.audio.transcriptions.create({
            file: fs.createReadStream(filePath),
            model: 'whisper-1'
        })
        return transcription.text
    } catch (error) {
        throw new Error(`Transcription failed: ${error.message}`)
    }
}

// Generate a summary with visual and audio transcription
import OpenAI from "openai";

const openai = new OpenAI({
    // eslint-disable-next-line no-undef
    apiKey: process.env.OPENAI_API_KEY
});

async function createVideoSummarization(frames, audioTranscription) {
    const frameObjects = frames.map(x => ({
        type: 'image_url',
        image_url: {
            url: `data:image/jpg; base64, ${x}`,
            detail: 'low'
        }
    }));

    const response = await openai.chat.completions.create({
        model: "gpt-4o",
        messages: [{
                role: "system",
                content: "You are generating a video summary. Please provide a summary of the video. Respond in Markdown."
            }
            ,
            {
                role: "user",
                content: [{
                        type: 'text',
                        text: "These are the frames from the video."
                    }
                    ,
                    ...frameObjects,
                    {
                        type: 'text',
                        text: `The audio transcription is: ${audioTranscription}`
                    }
                ],
            }
            ,
        ],
        temperature: 0,
    });
    console.log(response.choices[0].message.content);
    return response;
}

export { createVideoSummarization }

export async function saveData({ filename, audioTranscription, summary }) {
    const id = generateRandomID();
    const videoFilename = String(filename);
    const audioToText = String(audioTranscription);
    const videoSummary = String(summary);

    const packetInfo = [parseInt(id), videoFilename, audioToText, videoSummary];
    const saveStatus = await GridDB.insert(packetInfo, collectionDb);
    return saveStatus;
}

http://localhost:3000/summaries