How to Build a Quokkabot

A presentation at Microsoft Reactor in November 2022 in Perth WA, Australia by Amy Kapernick

I acknowledge the traditional owners of this land, the Whadjuk people of the Nyoongar nation. @amys_kapers

require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers

const twilio = require(‘twilio’) const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( `https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}` ) } else { message.body(‘This is not a quokka’) message.media( `https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}` ) } }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers

// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(`https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}`) } else { message.body(‘This is not a quokka’) message.media(`https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}`) } } context.log({results: JSON.stringify(results)}) }; // Finish Azure function @amys_kapers

{ } “results”: { “id”: “06f7250b-ffb3-47a1-9508-c928b8c54b87”, “project”: “a44bf78b-da55-4a95-bfdc-21ac780d2518”, “iteration”: “8a75c738-d658-4f63-8a2e-e18318f418f8”, “created”: “2022-10-12T06:31:47.662Z”, “predictions”: [ { “probability”: 0.99992716, “tagId”: “1f6223d5-79ce-4e6b-8f58-ddfbb4635246”, “tagName”: “Quokka”, “tagType”: “Regular” }, { “probability”: 0.00007281252, “tagId”: “7a44e503-b236-4991-bf44-2e08ca0e2356”, “tagName”: “Negative”, “tagType”: “Negative” } ] } @amys_kapers

“predictions”: [ { “probability”: 0.99992716, “tagId”: “1f6223d5-79ce-4e6b-8f58-ddfbb4635246”, “tagName”: “Quokka”, “tagType”: “Regular” }, { “probability”: 0.00007281252, “tagId”: “7a44e503-b236-4991-bf44-2e08ca0e2356”, “tagName”: “Negative”, “tagType”: “Negative” } ] @amys_kapers

“predictions”: [ { “probability”: 0.99992716, “tagName”: “Quokka”, }, { “probability”: 0.00007281252, “tagName”: “Negative”, } ] @amys_kapers

// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Get image classification results let outcome = {} results.predictions.forEach(tag => { if (tag.tagName == ‘Negative’) { outcome.negative = tag.probability } else if (tag.tagName == ‘Quokka’) { outcome.quokka = tag.probability } }) const quokka = `${(outcome.quokka * 100).toFixed(2)}%` const notQuokka = `${(outcome.negative * 100).toFixed(2)}%` if(outcome.negative > outcome.quokka) { message.body( `Sorry that doesn't look like a quokka\nQuokka: ${quokka}, Not Quokka: ${notQuokka}\nThat's pretty sad though, so here's a quokka` ) message.media(`https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}`) } else { message.body(`Yep that looks like a quokka!\nQuokka: ${quokka}, Not Quokka: ${notQuokka}`) } } else { // Send Quokka or not Quokka photo } }; // Finish Azure Function @amys_kapers

// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) message.media( `https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}` ) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { // Send Quokka } else { // Send not Quokka } } }; // Finish Azure Function @amys_kapers

Image Classification Object Detection Label: Cat Cat, Dog, Duck @amys_kapers

const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: `Re: ${body.subject}` } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = `${msg.html}` }; await sgMail.send(msg) @amys_kapers

quokkas.amyskapers.dev/results @amys_kapers

github.com/amykapernick/quokkas @amys_kapers

Amy Kapernick
@amyskapers

1 / 34

Quokkas are the happiest animal in Australia that surprisingly isn’t trying to kill you. But can you really be sure that what you’re looking at is a Quokka and not something more dangerous?

Let’s look at how we can build a quokka identifying chatbot, combining Twilio’s WhatsApp, MMS and SendGrid APIs with machine learning based image detection powered by Microsoft’s Custom Vision API.

Join me as we learn how to train and then link up an ML model across multiple communication channels to create an engaging, accessible chatbot. And of course lots of photos of quokkas.

Resources

The following resources were mentioned during the presentation or are useful additional information.

Quokkabot
Building Quokka on Demand

Stage one of Quokkabot was built on Twilio Functions, using the Twilio WhatsApp API. On request, the user can ask for a picture of a Quokka, and one is sent to them.
Building Quokkabot: Part 2

Building on the Functionality of Quokka on Demand, the next step was moving the function over to Azure Functions and hooking up Cognitive Services so that users could send in an image and it would detect if there was a Quokka in the photo
Building Quokkabot Part 3 - Moving to Azure Static Web Apps

Last week I got the chance to speak at Microsoft Build, and do a first-look demo on their newly announced featured - Azure Static Web Apps. And as I didn’t get access to Static Web Apps until they same time everyone else did (the day before my demo), I got to showcase how easy it was the first time you used it (ok, so maybe it was the second time that I’d used it).
Quokkabot Repo

How to Build a Quokkabot

Link for this presentation:

HTML code for embedding:

Share on social media:

Resources

Quokkabot

Building Quokka on Demand

Building Quokkabot: Part 2

Building Quokkabot Part 3 - Moving to Azure Static Web Apps

Quokkabot Repo