Bring pen and touch input to your Metro style apps with ink

1. PLAT-192C Bring pen and touch input to your Metro style apps with ink Annie Chowdhry Program Manager Microsoft Corporation Jay Pittman Developer Microsoft Corporation

2. Metro style apps in Windows 8 are so much better with Ink! Code for touch, get mouse and pen for free

3. Agenda • How Inking Works • Inking API architecture • Demos • Creating and rendering Ink to a canvas • Managing Ink from pen and touch input • Converting Ink to text (handwriting recognition) You’ll leave with examples of how to • Write an Ink-enabled Metro style app in Windows 8

4. demo Ink Pad AppOverview of Inking – capture, manipulate, and recognize

5. Recognize How Inking works Capture and Manipulate HI! Hi! Hit Hid hi! HI ! Reco API returns interpreted combination of strokes from recognizer (rich recognition results) Pointer API captures pen motion, passing coordinates to Ink API X,Y….Xn,Yn HI ! l Optional Ink API helps render and stores motion as ink. Ink manipulation. Ink API groups strokes and passes them to recognizer

6. Inking API architecture – execution flow Render Metro style app Input Obtains Raw Dataposition, id and pressure using MSPointer(JavaScript / ICoreWindow(XAML) and passes this to stroke builder: Begin/Append/End Stroke methods Windows.UI.Input.Inkingstroke builder,rendering helper,ink manipulation,handwriting recognition with rich recognition results with alernates,clipboard,serialization Renders Strokes render using rendering helper method:InkRenderingSegmentdoes not return packets but stroke rendering segments such as Bezier curves

7. Code Snippets • Register for events • ProcessPointerDown • ProcessPointerUpdate • ProcessPointerUp • Render everything

8. Register for events html:     <canvas id="myCanvas" width="1000" height="1000"></canvas> js: var canvas = document.getElementById("myCanvas"); var context = canvas.getContext("2d"); canvas.addEventListener("MSPointerDown", handlePointerDown, false); canvas.addEventListener("MSPointerMove", handlePointerMove, false); canvas.addEventListener("MSPointerUp", handlePointerUp, false); varinkManager = new Windows.UI.Input.Inking.InkManager();

9. ProcessPointerDown functionhandlePointerDown(evt) { if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1))) { evt.preventManipulation(); inkManager.processPointerDown(evt.currentPoint); context.beginPath(); context.moveTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y); } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch down } }

10. ProcessPointerUpdate functionhandlePointerMove(evt) { if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1))) { evt.preventManipulation(); inkManager.processPointerUpdate(evt.currentPoint); context.lineTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y); context.stroke();     } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch move } }

11. ProcessPointerUp functionhandlePointerUp(evt) { if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1))) { evt.preventManipulation(); inkManager.processPointerUp(evt.currentPoint); context.lineTo(evt.currentPoint.rawPosition.x, evt.currentPoint.rawPosition.y); context.stroke(); context.closePath(); } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { // Handle touch up } renderAllStrokes(); }

12. Render everything functionrenderAllStrokes() { inkManager.getStrokes().forEach(function (stroke) {var first = true; stroke.getRenderingSegments().forEach(function (segment) { if (first)             { context.moveTo(segment.position.x, segment.position.y);                 first = false;             } else {context.bezierCurveTo(segment.bezierControlPoint1.x, segment.bezierControlPoint1.y, segment.bezierControlPoint2.x, segment.bezierControlPoint2.y, segment.position.x, segment.position.y);             }         });     }); context.stroke(); context.closePath(); }

13. demos Ink Pad app Managing Ink from pen and touch input- Select- Move- Erase

14. Code Snippets • Select strokes using Touch • Moving selected strokes • Selecting and Erasing modes • Deleting selected strokes

15. Selecting individual strokes (using touch) functionhandlePointerDown(evt) {. . . else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { var hit = false; inkManager.getRecognitionResults().forEach(function (result) { if (inRect(evt.offsetX, evt.offsetY, result.boundingRect)) {result.getStrokes().forEach(function (stroke) { stroke.selected= true; hit = true; }); } }); if (hit) { evt.preventManipulation(); prevX= evt.offsetX; prevY= evt.offsetY; } } }

16. Moving selected strokes functionhandlePointerMove(evt) { if ((evt.pointerType === 3) || ((evt.pointerType === 4) && (evt.button === 1))) { // Handle pen move     } else if ((evt.pointerType === 2) || ((evt.pointerType === 4) && (evt.button === 2))) { evt.preventManipulation(); var shift = {x: evt.offsetX - prevX, y: evt.offsetY - prevY}; inkManager.moveSelected(shift); renderAllStrokes(); prevX = evt.offsetX; prevY= evt.offsetY; } }

17. Selecting mode (Lasso) and erasing mode inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.selecting; inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.erasing; inkManager.mode = Windows.UI.Input.Inking.InkManipulationMode.inking; // These 3 modes all use the same down/move/up handlers inkManager.deleteSelected();

18. demo Ink Pad App Converting Ink to text (handwriting recognition) and searching

19. Code Snippets • Recognition • Find and select all strokes

20. Recognition function recognize() { inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.all).then( function(results)     { inkManager.updateRecognitionResults(results);     }); }

21. Find and select all strokes functionrecognizeThenFindAndSelect(target) { inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.all).then(function (results) { results.forEach(function (result) { result.getTextCandidates().forEach(function(candidate) { if (target.toLowerCase() === candidate.toLowerCase()) { result.getStrokes().forEach(function (stroke) { stroke.selected = true;                     });                 }             });         });     }); }

22. The new Ink API has been optimized to enable you with easier ramp-up while still covering the most important Metro style app scenarios.

23. Call to action • Capture Ink • Manipulate Ink • Recognize Ink Create Ink enabled Metro style apps

24. Related sessions • [APP-185T] Make great Metro style apps that are touch-optimized using HTML5 • [APP-186T] Build advanced touch apps in Windows 8 • [APP-395T] Designing Metro style: principles and personality • [APP-740T] Metro style apps using HTML5 from start to finish

25. Further reading and documentation • Quickstart: Capturing and displaying ink • Metro style apps > Learn > Developing basic Metro style apps > Developing basic apps (JavaScript) > Responding to user input • Ink App sample in Windows SDK for Metro style apps • Contact: • For questions, please visit the forums on the Windows Dev Center at http://forums.dev.windows.com • For best response, please include the Build Session # in the title of your post

26. thank you Feedback and questions http://forums.dev.windows.com Session feedbackhttp://bldw.in/SessionFeedback

27. © 2011 Microsoft Corporation. All rights reserved. Microsoft, Windows, Windows Vista and other product names are or may be registered trademarks and/or trademarks in the U.S. and/or other countries. The information herein is for informational purposes only and represents the current view of Microsoft Corporation as of the date of this presentation. Because Microsoft must respond to changing market conditions, it should not be interpreted to be a commitment on the part of Microsoft, and Microsoft cannot guarantee the accuracy of any information provided after the date of this presentation. MICROSOFT MAKES NO WARRANTIES, EXPRESS, IMPLIED OR STATUTORY, AS TO THE INFORMATION IN THIS PRESENTATION.

28. Metro style app built for Windows using JavaScript / C++ or C# runtime environment Inking API Architecture – Binary perspective Metro style app obtains position, id, & pressure from MSPointer (JavaScript) / ICoreWindow (C++/C#) manages ink with calls to Windows.UI.Input.Inking renders using HTML5 Canvas (JavaScript), D2D and DUI Windows runtime Windows.UI.Input.Inkingnamespace for ink management Windows.Graphicsnamespace for rendering IE events for JavaScript / Windows.Foundationnamespace for ICoreWindow (C++ / C#)

29. Save functionsaveToFile() { var picker = new Windows.Storage.Pickers.FileSavePicker(); picker.pickSingleFileAsync().then(function (file) { file.openAsync(Windows.Storage.FileAccessMode.readWrite).then(function (stream) { varoutputStream = fileStream.getOutputStreamAt(0); inkManager.saveAsync(outputStream).then(function()             { outputStream.flushAsync().then(function() {                 });             });         });     }); }

30. Load functionloadFromFile() { var picker = new Windows.Storage.Pickers.FileOpenPicker(); picker.pickSingleFileAsync().then(function (file) { file.openAsync(Windows.Storage.FileAccessMode.read).then(function (stream) { varinputStream = fileStream.getInputStreamAt(0); inkManager.Load(inputStream); renderAllStrokes();         });     }); }

31. Copy and Paste functionpasteFromClipboard(point) { inkManager.pasteFromClipboard(point); } functioncopySelectedToClipboard() { inkManager.copySelectedToClipboard(); }

32. Select with Recent Recognition functionrecognizeRecentThenFindAndSelect(target) { inkManager.recognizeAsync(Windows.UI.Input.Inking.InkRecognitionTarget.recent).then(function (results)     { inkManager.updateRecognitionResults(results); inkManager.getRecognitionResults().forEach(function (result)         { result.getTextCandidates().forEach(function (candidate)            { if (target.toLowerCase() === candidate. toLowerCase()) { result.getStrokes().forEach(function (stroke) { stroke.selected = true; });                 }             }); });     }); }