Inference with C# BERT NLP Deep Learning and ONNX Runtime

本文介绍: Inf e rence wi th C# BERT NLP Deep Learnin g and ONNX Runtime

效果

Inference wi th C# BERT NLP Deep Learning and ONNX Runtime

效果

测试一

Con text ：Bo b i s wal k ing t hr ough the wo ods collect ing blue b err ie s and s t rawberr ie s to make a p ie.

Ques t ion ：What i s h i s name?

测试二

Con text ：Bob i s wal k ing through the woods collect ing blueber r ies and strawber r ies to make a pie.

Question ：What will he br ing home?

测试三

Context ：Bob is wal k ing through the woods collect ing blueber r ies and strawber r ies to make a pie.

Question ：Where is Bob?

Inputs
————————-
name：unique _ids_raw _output ___9:0
tensor：Int64[-1]
name：segment_ids:0
tensor：Int64[-1, 256]
name：input _mask:0
tenso r：Int64[-1, 256]
name：input _ids:0
tensor：Int64[-1, 256]
—————————————————————

Outputs
————————-
name：unstack:1
tensor：Float[-1, 256]
name：unstack:0
tensor：Float[-1, 256]
name：unique _ids:0
tensor：Int64[-1]
—————————————————————

项目

代码

using BERTTokenizers;
using Mic ro soft.ML.Onn xRuntime;
using Sy ste m;
using Syste m.Collect ions.Generic;
using Syste m.Data;
using Syste m.Diagnos ti cs;
using Syste m.Linq;
using Syste m.Windows.Forms;

namespace Inference_wi th_C__BERT_NLP_Deep_Learning_and_ONNX_Runtime
{
public struct BertInput
{
public long[] InputIds { get; set; }
public long[] InputMask { get; set; }
public long[] SegmentIds { get; set; }
public long[] UniqueIds { get; set; }
}

public partial class Form1 : Form
{
public Form1()
{
Initi alizeComponent();
}

RunOptions runOptions;
InferenceSession session;
BertUncasedLargeTokenizer tokenizer;
Stop watch stopWatc h = new Stop watch();

private void Form1_Load(object sender, EventArgs e)
{
string modelPath = “bertsquad-10.onnx“;
runOptions = new RunOptions();
session = new InferenceSession(modelPath);
tokenizer = new BertUncasedLargeTokenizer();
}

int MaxAnswerLength = 30;
int bestN = 20;

private void button1_Click(object sender, EventArgs e)
{
txt_answer.Text = “”;
Appli cation.DoEvents();

string question = txt_question.Text.Trim();
string context = txt_context.Text.Trim();

// Get the sentence tokens.
var tokens = tokenizer.Tokenize(question, context);

// Encode the sentence and pass in the count of the tokens in the sentence.
var encoded = tokenizer.Encode(tokens.Count(), question, context);

var padding = Enumerable
.Repeat(0L, 256 – tokens.Count)
.ToList();

var bertInput = new BertInput()
{
InputIds = encoded.Sel ect(t => t.InputIds).Concat(padding).ToArray(),
InputMask = encoded.Sel ect(t => t.AttentionMask).Concat(padding).ToArray(),
SegmentIds = encoded.Sel ect(t => t.TokenTypeIds).Concat(padding).ToArray(),
UniqueIds = new long[] { 0 }
};

// Create input tensors over the input data.
var inputIdsOrtValue = OrtValue.CreateTensorValueFromMemo ry(bertInput.InputIds,
new long[] { 1, bertInput.InputIds.Length });

var inputMaskOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.InputMask,
new long[] { 1, bertInput.InputMask.Length });

var segmentIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.SegmentIds,
new long[] { 1, bertInput.SegmentIds.Length });

var uniqueIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.UniqueIds,
new long[] { bertInput.UniqueIds.Length });

var inputs = new Dictionary<string, OrtValue>
{
{ “unique_ids_raw_output ___9:0″, uniqueIdsOrtValue },
{ “segment_ids:0″, segmentIdsOrtValue},
{ “input_mask:0″, inputMaskOrtValue },
{ “input_ids:0”, inputIdsOrtValue }
};

stopWatch.Restart();
// Run session and send the input data in to get inference output.
var output = session.Run(runOptions, inputs, session.OutputNames);
stopWatch.Stop();

var startLogits = output[1].GetTensorDataAsSpan<float>();

var endLogits = output[0].GetTensorDataAsSpan<float>();

var uniqueIds = output[2].GetTensorDataAsSpan<long>();

var contextStart = tokens.FindInde x(o => o.Token == “[SEP]”);

var bestStartLogits = startLogits.ToArray()
.Sel ect((logit, index) => (Logit: logit, Inde x: index))
.OrderByDesc ending(o => o.Logit)
.Take(bestN);

var bestEndLogits = endLogits.ToArray()
.Select((logit, index) => (Logit: logit, Index: index))
.OrderByDesc ending(o => o.Logit)
.Take(bestN);

var bestResul tsWithScore = bestStartLogits
.SelectMany(startLogit =>
bestEndLogits
.Select(endLogit =>
(
StartLogit: startLogit.Index,
EndLogit: endLogit.Index,
Score: startLogit.Logit + endLogit.Logit
)
)
)
.Where(entry => !(entry.EndLogit < entry.StartLogit || entry.EndLogit – entry.StartLogit > MaxAnswerLength || entry.StartLogit == 0 && entry.EndLogit == 0 || entry.StartLogit < contextStart))
.Take(bestN);

var (item, probabi lity) = bestResultsWithScore
.Soft max(o => o.Score)
.OrderByDescending(o => o.Probabi lity)
.FirstOrDefa ult();

int startIndex = item.StartLogit;
int endIndex = item.EndLogit;

var pre dictedTokens = tokens
.Skip(startIndex)
.Take(endIndex + 1 – startIndex)
.Select(o => tokenizer.IdToToken((int)o.Voca bularyIndex))
.ToList();

// Print the result.
string answer = “answer：” + String.Join(” “, StitchSentenceBackTogether(pre dictedTokens))
+ “rnprobabi lity：” + probabi lity
+ $”rn推理耗时：{stopWatch.Elap sedMilliseconds}毫秒”;

txt_answer.Text = answer;
Console.WriteLine(answer);

}

private List<string> StitchSentenceBackTogether(List<string> tokens)
{
var currentToken = string.Empty;

tokens.Reverse();

var tokensStitched = new List<string>();

foreach (var token in tokens)
{
if (!token.StartsWith(“##”))
{
currentToken = token + currentToken;
tokensStitched.Add(currentToken);
currentToken = string.Empty;
}
else
{
currentToken = token.Replace(“##”, “”) + currentToken;
}
}

tokensStitched.Reverse();

return tokensStitched;
}
}
}

using BERTTokenizers;
using Microsoft.ML.OnnxRuntime;
using System;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Linq;
using System.Windows.Forms;

namespace Inference_with_C__BERT_NLP_Deep_Learning_and_ONNX_Runtime
{
    public struct BertInput
    {
        public long[] InputIds { get; set; }
        public long[] InputMask { get; set; }
        public long[] SegmentIds { get; set; }
        public long[] UniqueIds { get; set; }
    }

    public partial class Form1 : Form
    {
        public Form1()
        {
            InitializeComponent();
        }

        RunOptions runOptions;
        InferenceSession session;
        BertUncasedLargeTokenizer tokenizer;
        Stopwatch stopWatch = new Stopwatch();

        private void Form1_Load(object sender, EventArgs e)
        {
            string modelPath = "bertsquad-10.onnx";
            runOptions = new RunOptions();
            session = new InferenceSession(modelPath);
            tokenizer = new BertUncasedLargeTokenizer();
        }

        int MaxAnswerLength = 30;
        int bestN = 20;

        private void button1_Click(object sender, EventArgs e)
        {
            txt_answer.Text = "";
            Application.DoEvents();

            string question = txt_question.Text.Trim();
            string context = txt_context.Text.Trim();

            // Get the sentence tokens.
            var tokens = tokenizer.Tokenize(question, context);

            // Encode the sentence and pass in the count of the tokens in the sentence.
            var encoded = tokenizer.Encode(tokens.Count(), question, context);

            var padding = Enumerable
              .Repeat(0L, 256 - tokens.Count)
              .ToList();

            var bertInput = new BertInput()
            {
                InputIds = encoded.Select(t => t.InputIds).Concat(padding).ToArray(),
                InputMask = encoded.Select(t => t.AttentionMask).Concat(padding).ToArray(),
                SegmentIds = encoded.Select(t => t.TokenTypeIds).Concat(padding).ToArray(),
                UniqueIds = new long[] { 0 }
            };

            // Create input tensors over the input data.
            var inputIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.InputIds,
                  new long[] { 1, bertInput.InputIds.Length });

            var inputMaskOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.InputMask,
                  new long[] { 1, bertInput.InputMask.Length });

            var segmentIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.SegmentIds,
                  new long[] { 1, bertInput.SegmentIds.Length });

            var uniqueIdsOrtValue = OrtValue.CreateTensorValueFromMemory(bertInput.UniqueIds,
                  new long[] { bertInput.UniqueIds.Length });

            var inputs = new Dictionary<string, OrtValue>
              {
                  { "unique_ids_raw_output___9:0", uniqueIdsOrtValue },
                  { "segment_ids:0", segmentIdsOrtValue},
                  { "input_mask:0", inputMaskOrtValue },
                  { "input_ids:0", inputIdsOrtValue }
              };

            stopWatch.Restart();
            // Run session and send the input data in to get inference output. 
            var output = session.Run(runOptions, inputs, session.OutputNames);
            stopWatch.Stop();

            var startLogits = output[1].GetTensorDataAsSpan<float>();

            var endLogits = output[0].GetTensorDataAsSpan<float>();

            var uniqueIds = output[2].GetTensorDataAsSpan<long>();

            var contextStart = tokens.FindIndex(o => o.Token == "[SEP]");

            var bestStartLogits = startLogits.ToArray()
                .Select((logit, index) => (Logit: logit, Index: index))
                .OrderByDescending(o => o.Logit)
                .Take(bestN);

            var bestEndLogits = endLogits.ToArray()
                .Select((logit, index) => (Logit: logit, Index: index))
                .OrderByDescending(o => o.Logit)
                .Take(bestN);

            var bestResultsWithScore = bestStartLogits
                .SelectMany(startLogit =>
                    bestEndLogits
                    .Select(endLogit =>
                        (
                            StartLogit: startLogit.Index,
                            EndLogit: endLogit.Index,
                            Score: startLogit.Logit + endLogit.Logit
                        )
                     )
                )
                .Where(entry => !(entry.EndLogit < entry.StartLogit || entry.EndLogit - entry.StartLogit > MaxAnswerLength || entry.StartLogit == 0 &amp;&amp; entry.EndLogit == 0 || entry.StartLogit < contextStart))
                .Take(bestN);

            var (item, probability) = bestResultsWithScore
                .Softmax(o => o.Score)
                .OrderByDescending(o => o.Probability)
                .FirstOrDefault();

            int startIndex = item.StartLogit;
            int endIndex = item.EndLogit;

            var predictedTokens = tokens
                          .Skip(startIndex)
                          .Take(endIndex + 1 - startIndex)
                          .Select(o => tokenizer.IdToToken((int)o.VocabularyIndex))
                          .ToList();

            // Print the result.
            string answer = "answer：" + String.Join(" ", StitchSentenceBackTogether(predictedTokens))
                + "rnprobability：" + probability
                + $"rn推理耗时：{stopWatch.ElapsedMilliseconds}毫秒";

            txt_answer.Text = answer;
            Console.WriteLine(answer);

        }

        private List<string> StitchSentenceBackTogether(List<string> tokens)
        {
            var currentToken = string.Empty;

            tokens.Reverse();

            var tokensStitched = new List<string>();

            foreach (var token in tokens)
            {
                if (!token.StartsWith("##"))
                {
                    currentToken = token + currentToken;
                    tokensStitched.Add(currentToken);
                    currentToken = string.Empty;
                }
                else
                {
                    currentToken = token.Replace("##", "") + currentToken;
                }
            }

            tokensStitched.Reverse();

            return tokensStitched;
        }
    }
}