How to Build a Quokkabot
A presentation at NDC Sydney in in Sydney NSW, Australia by Amy Kapernick
How to Build a Quokkabot @amys_kapers
How to Build a Quokkabot @amys_kapers
I acknowledge the traditional owners of this land, the Gadigal people of the Eora nation. @amys_kapers
@amys_kapers
How to Build a Quokkabot @amys_kapers
@amys_kapers
@amys_kapers
@amys_kapers
@amys_kapers
Demo Time 🤞 @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
require(‘dotenv’).config() const twilio = require(‘twilio’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() message.body(“Welcome to Quokkabot”) }; context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, }) @amys_kapers
const twilio = require(‘twilio’)
const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else { message.body(‘This is not a quokka’) message.media( https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug} ) }
};
context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, })
@amys_kapers
const twilio = require(‘twilio’)
const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else { message.body(‘This is not a quokka’) message.media( https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug} ) }
};
context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, })
@amys_kapers
const twilio = require(‘twilio’)
const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else { message.body(‘This is not a quokka’) message.media( https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug} ) }
};
context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, })
@amys_kapers
const twilio = require(‘twilio’)
const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else { message.body(‘This is not a quokka’) message.media( https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug} ) }
};
context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, })
@amys_kapers
const twilio = require(‘twilio’)
const { randomImage, quokkas, notQuokkas } = require(‘../_data/photos’) module.exports = async function (context) { const MessagingResponse = twilio.twiml.MessagingResponse const twiml = new MessagingResponse() const message = twiml.message() const msgParams = new URLSearchParams(context.req.body) const msgText = msgParams.get(‘Body’) if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else { message.body(‘This is not a quokka’) message.media( https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug} ) }
};
context.done(null, { status: 200, body: message.toString(), headers: { ‘Content-Type’: ‘text/xml’ }, })
@amys_kapers
Try it yourself! @amys_kapers
@amys_kapers
// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const
key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION
const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(‘This is not a quokka’) message.media(https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}) } }
context.log({results: JSON.stringify(results)}) };
// Finish Azure function
@amys_kapers
// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const
key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION
const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(‘This is not a quokka’) message.media(https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}) } }
context.log({results: JSON.stringify(results)}) };
// Finish Azure function
@amys_kapers
// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const
key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION
const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(‘This is not a quokka’) message.media(https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}) } }
context.log({results: JSON.stringify(results)}) };
// Finish Azure function
@amys_kapers
// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const
key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION
const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(‘This is not a quokka’) message.media(https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}) } }
context.log({results: JSON.stringify(results)}) };
// Finish Azure function
@amys_kapers
// Existing imports here const { PredictionAPIClient } = require(‘@azure/cognitiveservices-customvision-prediction’) const { ApiKeyCredentials } = require(‘@azure/ms-rest-js’) const const const const
key = process.env.API_KEY endpoint = process.env.ENDPOINT projectId = process.env.PROJECT_ID iteration = process.env.ITERATION
const credentials = new ApiKeyCredentials({ inHeader: { “Prediction-key”: key } }) const predictor = new PredictionAPIClient(credentials, endpoint) module.exports = async function (context) { // Existing code here const image = msgParams.get(‘MediaUrl0’) let results = false if(image) { results = await predictor.classifyImageUrl(projectId, iteration, { url: image }) } else { if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(‘This is not a quokka’) message.media(https://quokkas.amyskapers.dev/img/not_quokkas/${randomImage(notQuokkas).slug}) } }
context.log({results: JSON.stringify(results)}) };
// Finish Azure function
@amys_kapers
{ } “results”: { “id”: “06f7250b-ffb3-47a1-9508-c928b8c54b87”, “project”: “a44bf78b-da55-4a95-bfdc-21ac780d2518”, “iteration”: “8a75c738-d658-4f63-8a2e-e18318f418f8”, “created”: “2022-10-12T06:31:47.662Z”, “predictions”: [ { “probability”: 0.99992716, “tagId”: “1f6223d5-79ce-4e6b-8f58-ddfbb4635246”, “tagName”: “Quokka”, “tagType”: “Regular” }, { “probability”: 0.00007281252, “tagId”: “7a44e503-b236-4991-bf44-2e08ca0e2356”, “tagName”: “Negative”, “tagType”: “Negative” } ] } @amys_kapers
“predictions”: [ { “probability”: 0.99992716, “tagId”: “1f6223d5-79ce-4e6b-8f58-ddfbb4635246”, “tagName”: “Quokka”, “tagType”: “Regular” }, { “probability”: 0.00007281252, “tagId”: “7a44e503-b236-4991-bf44-2e08ca0e2356”, “tagName”: “Negative”, “tagType”: “Negative” } ] @amys_kapers
“predictions”: [ { “probability”: 0.99992716, “tagName”: “Quokka”, }, { “probability”: 0.00007281252, “tagName”: “Negative”, } ] @amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Get image classification results let outcome = {} results.predictions.forEach(tag => { if (tag.tagName == ‘Negative’) { outcome.negative = tag.probability } else if (tag.tagName == ‘Quokka’) { outcome.quokka = tag.probability } }) const quokka = ${(outcome.quokka * 100).toFixed(2)}% const notQuokka = ${(outcome.negative * 100).toFixed(2)}% if(outcome.negative > outcome.quokka) { message.body( Sorry that doesn't look like a quokka\nQuokka: ${quokka}, Not Quokka: ${notQuokka}\nThat's pretty sad though, so here's a quokka ) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(Yep that looks like a quokka!\nQuokka: ${quokka}, Not Quokka: ${notQuokka}) }
} else { // Send Quokka or not Quokka photo } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Get image classification results let outcome = {} results.predictions.forEach(tag => { if (tag.tagName == ‘Negative’) { outcome.negative = tag.probability } else if (tag.tagName == ‘Quokka’) { outcome.quokka = tag.probability } }) const quokka = ${(outcome.quokka * 100).toFixed(2)}% const notQuokka = ${(outcome.negative * 100).toFixed(2)}% if(outcome.negative > outcome.quokka) { message.body( Sorry that doesn't look like a quokka\nQuokka: ${quokka}, Not Quokka: ${notQuokka}\nThat's pretty sad though, so here's a quokka ) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(Yep that looks like a quokka!\nQuokka: ${quokka}, Not Quokka: ${notQuokka}) }
} else { // Send Quokka or not Quokka photo } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Get image classification results let outcome = {} results.predictions.forEach(tag => { if (tag.tagName == ‘Negative’) { outcome.negative = tag.probability } else if (tag.tagName == ‘Quokka’) { outcome.quokka = tag.probability } }) const quokka = ${(outcome.quokka * 100).toFixed(2)}% const notQuokka = ${(outcome.negative * 100).toFixed(2)}% if(outcome.negative > outcome.quokka) { message.body( Sorry that doesn't look like a quokka\nQuokka: ${quokka}, Not Quokka: ${notQuokka}\nThat's pretty sad though, so here's a quokka ) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(Yep that looks like a quokka!\nQuokka: ${quokka}, Not Quokka: ${notQuokka}) }
} else { // Send Quokka or not Quokka photo } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Get image classification results let outcome = {} results.predictions.forEach(tag => { if (tag.tagName == ‘Negative’) { outcome.negative = tag.probability } else if (tag.tagName == ‘Quokka’) { outcome.quokka = tag.probability } }) const quokka = ${(outcome.quokka * 100).toFixed(2)}% const notQuokka = ${(outcome.negative * 100).toFixed(2)}% if(outcome.negative > outcome.quokka) { message.body( Sorry that doesn't look like a quokka\nQuokka: ${quokka}, Not Quokka: ${notQuokka}\nThat's pretty sad though, so here's a quokka ) message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug}) } else { message.body(Yep that looks like a quokka!\nQuokka: ${quokka}, Not Quokka: ${notQuokka}) }
} else { // Send Quokka or not Quokka photo } };
// Finish Azure Function
@amys_kapers
Try it yourself! @amys_kapers
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { // Send Quokka } else { // Send not Quokka } } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { // Send Quokka } else { // Send not Quokka } } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { // Send Quokka } else { // Send not Quokka } } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) message.media( https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug} ) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { // Send Quokka } else { // Send not Quokka } } };
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here
if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) } else if (RegExp(‘fact’, ‘i’).test(msgText)) { message.body(randomFacts(facts)) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) } else { message.body(Welcome to Quokkabot! I can do a bunch of different things that have to do with quokkas.\nNeed a picture of a quokka? Just ask me\nNot sure if you've seen a quokka? Send me a picture and I'll tell you if there's a quokka in it) } } };
message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug})
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here
if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) } else if (RegExp(‘fact’, ‘i’).test(msgText)) { message.body(randomFacts(facts)) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) } else { message.body(Welcome to Quokkabot! I can do a bunch of different things that have to do with quokkas.\nNeed a picture of a quokka? Just ask me\nNot sure if you've seen a quokka? Send me a picture and I'll tell you if there's a quokka in it) } } };
message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug})
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here
if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) } else if (RegExp(‘fact’, ‘i’).test(msgText)) { message.body(randomFacts(facts)) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) } else { message.body(Welcome to Quokkabot! I can do a bunch of different things that have to do with quokkas.\nNeed a picture of a quokka? Just ask me\nNot sure if you've seen a quokka? Send me a picture and I'll tell you if there's a quokka in it) } } };
message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug})
// Finish Azure Function
@amys_kapers
// Existing imports here module.exports = async function (context) { // Existing code here
if(image) { // Check Image } else { if(RegExp(/error|issue|wrong/, ‘i’).test(msgText)) { message.body(‘Sorry about that, here is a quokka to cheer you up’) } else if (RegExp(‘fact’, ‘i’).test(msgText)) { message.body(randomFacts(facts)) } else if(RegExp(‘quokka’, ‘i’).test(msgText)) { message.body(‘This is a quokka’) } else { message.body(Welcome to Quokkabot! I can do a bunch of different things that have to do with quokkas.\nNeed a picture of a quokka? Just ask me\nNot sure if you've seen a quokka? Send me a picture and I'll tell you if there's a quokka in it) } } };
message.media(https://quokkas.amyskapers.dev/img/quokkas/${randomImage(quokkas).slug})
// Finish Azure Function
@amys_kapers
Demo Time 🤞 @amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results }
msg.html = formatResults(results) else { // Send a Quokka, fact or information } msg.html = ${msg.html}
};
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
const sgMail = require(‘@sendgrid/mail’) const parseMultipartFormData = require(‘@anzp/azure-function-multipart’).default module.exports = async function (context, req) { const { fields, files } = await parseMultipartFormData(req) const body = {} const image = (files && files.length) && files[0].bufferFile fields.forEach(field => { body[field.name] = field.value }) sgMail.setApiKey(process.env.SENDGRID_API_KEY) let msg = { to: body.from, from: { name: ‘Quokkabot’, email: ‘[email protected]’, }, subject: Re: ${body.subject} } if (image) { const results = results = await predictor. classifyImage(projectId, iteration, image) // Format and return image results msg.html = formatResults(results) } else { // Send a Quokka, fact or information } msg.html = ${msg.html} };
await sgMail.send(msg)
@amys_kapers
Try it yourself! @amys_kapers
[email protected] @amys_kapers
quokkas.amyskapers.dev @amys_kapers
quokkas.amyskapers.dev/results @amys_kapers
What next? @amys_kapers
github.com/amykapernick/quokkas @amys_kapers
github.com/amykapernick/quokkas @amys_kapers
But why? @amys_kapers
Thank You 👏 @amys_kapers
Questions? @amys_kapers
Quokkas are the happiest animal in Australia that surprisingly isn’t trying to kill you. But can you really be sure that what you’re looking at is a Quokka and not something more dangerous?
Let’s look at how we can build a quokka identifying chatbot, combining Twilio’s WhatsApp, MMS and SendGrid APIs with machine learning based image detection powered by Microsoft’s Custom Vision API.
Join me as we learn how to train and then link up an ML model across multiple communication channels to create an engaging, accessible chatbot. And of course lots of photos of quokkas.
Resources
The following resources were mentioned during the presentation or are useful additional information.
-
Quokkabot
-
Building Quokka on Demand
Stage one of Quokkabot was built on Twilio Functions, using the Twilio WhatsApp API. On request, the user can ask for a picture of a Quokka, and one is sent to them.
-
Building Quokkabot: Part 2
Building on the Functionality of Quokka on Demand, the next step was moving the function over to Azure Functions and hooking up Cognitive Services so that users could send in an image and it would detect if there was a Quokka in the photo
-
Building Quokkabot Part 3 - Moving to Azure Static Web Apps
Last week I got the chance to speak at Microsoft Build, and do a first-look demo on their newly announced featured - Azure Static Web Apps. And as I didn’t get access to Static Web Apps until they same time everyone else did (the day before my demo), I got to showcase how easy it was the first time you used it (ok, so maybe it was the second time that I’d used it).
-
Quokkabot Repo
Buzz and feedback
Here’s what was said about this presentation on social media.
