diff --git a/samples/nbg_runner_pybind/.clang-format b/samples/nbg_runner_pybind/.clang-format new file mode 100644 index 000000000..4a6f71760 --- /dev/null +++ b/samples/nbg_runner_pybind/.clang-format @@ -0,0 +1,4 @@ +BasedOnStyle: Google +PointerAlignment: Left +DerivePointerAlignment: false +ColumnLimit: 80 \ No newline at end of file diff --git a/samples/nbg_runner_pybind/.clang-tidy b/samples/nbg_runner_pybind/.clang-tidy new file mode 100644 index 000000000..8e388cd5c --- /dev/null +++ b/samples/nbg_runner_pybind/.clang-tidy @@ -0,0 +1,69 @@ +Checks: > + -*, + bugprone-*, + -bugprone-easily-swappable-parameters, + performance-*, + portability-*, + readability-*, + modernize-*, + -modernize-loop-convert, + -modernize-use-trailing-return-type, + clang-analyzer-*, + misc-*, + google-*, + +HeaderFilterRegex: "(src/).*\\.(hpp|h)$" + +CheckOptions: + - key: readability-identifier-naming.NamespaceCase + value: snake_case + - key: readability-identifier-naming.ClassCase + value: CamelCase + - key: readability-identifier-naming.StructCase + value: CamelCase + - key: readability-identifier-naming.EnumCase + value: CamelCase + - key: readability-identifier-naming.TemplateParameterCase + value: CamelCase + - key: readability-identifier-naming.FunctionCase + value: lower_case + - key: readability-identifier-naming.VariableCase + value: lower_case + - key: readability-identifier-naming.ClassMemberCase + value: lower_case + - key: readability-identifier-naming.ClassMemberSuffix + value: _ + - key: readability-identifier-naming.PrivateMemberCase + value: lower_case + - key: readability-identifier-naming.PrivateMemberSuffix + value: _ + - key: readability-identifier-naming.ClassMethodCase + value: lower_case + - key: readability-identifier-naming.EnumConstantCase + value: CamelCase + - key: readability-identifier-naming.EnumConstantPrefix + value: k + - key: readability-identifier-naming.ConstexprVariableCase + value: CamelCase + - key: readability-identifier-naming.ConstexprVariablePrefix + value: k + - key: readability-identifier-naming.GlobalConstantCase + value: CamelCase + - key: readability-identifier-naming.GlobalConstantPrefix + value: k + - key: readability-identifier-naming.MemberConstantCase + value: CamelCase + - key: readability-identifier-naming.MemberConstantPrefix + value: k + - key: readability-identifier-naming.StaticConstantCase + value: CamelCase + - key: readability-identifier-naming.StaticConstantPrefix + value: k + - key: readability-identifier-length.IgnoredVariableNames + value: "^c|d|e|m|op|fd|it$" + - key: readability-identifier-length.IgnoredParameterNames + value: "^c|d|e|m|op|fd|it$" + - key: readability-function-cognitive-complexity.IgnoreMacros + value: true + +FormatStyle: file \ No newline at end of file diff --git a/samples/nbg_runner_pybind/.gitignore b/samples/nbg_runner_pybind/.gitignore new file mode 100644 index 000000000..b364b874e --- /dev/null +++ b/samples/nbg_runner_pybind/.gitignore @@ -0,0 +1,15 @@ +[Bb]uild/ +*_build/ +.vscode/ +.cache/ +__pycache__/ +*.pyc +temp/ +tmp/ +*.o +*.a +*.lib +*.dll +*.so +*.nb +.env \ No newline at end of file diff --git a/samples/nbg_runner_pybind/CMakeLists.txt b/samples/nbg_runner_pybind/CMakeLists.txt new file mode 100644 index 000000000..94c583f91 --- /dev/null +++ b/samples/nbg_runner_pybind/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.18) +project(nbg_runner LANGUAGES C CXX) + +# Set C/C++ standard. +set(CMAKE_C_STANDARD 11) +set(CMAKE_C_STANDARD_REQUIRED ON) +set(CMAKE_C_EXTENSIONS OFF) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# Enable warnings as errors. +add_compile_options(-Wall -Werror) + +# Add custom CMake modules. +list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/modules) + +# Find dependencies. +include(FindPyBind11) +include(FindVivanteSDK) + +add_subdirectory(src) \ No newline at end of file diff --git a/samples/nbg_runner_pybind/README.md b/samples/nbg_runner_pybind/README.md new file mode 100644 index 000000000..dd7c51061 --- /dev/null +++ b/samples/nbg_runner_pybind/README.md @@ -0,0 +1,84 @@ +# VSI NBG Runner Python Binding + +This project is a python package that wraps OpenVX API using pybind11. It provides simple python API to load, query and run model NBG. + +## Installation + +```cmake +cmake -B build -DVIVANTE_SDK_DIR=${VIV_SDK_INSTALL_PATH} +cmake --build build +``` + +The built python binding lib can be found at +`build/src/_nbg_runner.cpython-{python_version}-{platform}.so`. Place the lib into `python/nbg_runner/_binding/`. + +## Usage + +### Tensor Info + +| Field | Type | Value Sample | +|:-----------------:|:-----------------:|:-------------:| +| rank | int | 4 | +| shape | Tuple[int, ...] | (1,3,224,224) | +| dtype | str | "uint8" | +| qtype | str | "affine" | +| scale | float | 0.007874 | +| zero_point | int | 128 | +| fixed_point_pos | int | 0 | + +- `shape` is in C-style row major order, which is consistent with NumPy. + +### Set Environment Vars + +```shell +# Set HW target If the driver is compiled with vsimulator. +VSIMULATOR_CONFIG=VIP9000ULSI_PID0XBA +# Locate the OVX driver. +VIVANTE_SDK_DIR=${VIV_SDK_INSTALL_PATH} +LD_LIBRARY_PATH=${VIVANTE_SDK_DIR}/[lib|lib64|drivers] +# Set PYTHONPATH to the dir containing nbg_runner module. +PYTHONPATH=${workspaceFolder}/python +``` + +### Example + +See detailed examples in `examples/*.py` + +```python +from nbg_runner import OVXExecutor + +# Load a model NBG file. +executor = OVXExecutor("path/to/model.nbg") + +# Query model I/O tensors count. +num_inputs = executor.get_num_inputs() +num_outputs = executor.get_num_outputs() + +# Get I/O tensor info by index. +input_info = executor.get_input_info(0) +output_info = executor.get_output_info(0) + +# Or get all I/O tensors infos at once. +input_infos = executor.get_input_infos() +output_infos = executor.get_output_infos() + +# Prepare inputs. +input_tensors: List[NDArray] = ... + +# Set input tensor by index. +for i, input_tensor in enumerate(input_tensors): + executor.set_input(i, input_tensor) + +# Or set all input tensors at once. +executor.set_inputs(input_tensors) + +# Run inference. +executor.run() + +# Get output tensor by index. +for i in range(num_outputs): + output_tensor = executor.get_output(i) + +# Or get all output tensors at once. +output_tensors = executor.get_outputs() +``` diff --git a/samples/nbg_runner_pybind/cmake/modules/FindPyBind11.cmake b/samples/nbg_runner_pybind/cmake/modules/FindPyBind11.cmake new file mode 100644 index 000000000..c755fff67 --- /dev/null +++ b/samples/nbg_runner_pybind/cmake/modules/FindPyBind11.cmake @@ -0,0 +1,13 @@ +# Try to use installed pybind11 CMake module. +find_package(pybind11) + +if(NOT ${pybind11_FOUND}) + include(FetchContent) + + FetchContent_Declare( + pybind11 + GIT_REPOSITORY "https://github.com/pybind/pybind11.git" + GIT_TAG "v2.11.1" + ) + FetchContent_MakeAvailable(pybind11) +endif() \ No newline at end of file diff --git a/samples/nbg_runner_pybind/cmake/modules/FindVivanteSDK.cmake b/samples/nbg_runner_pybind/cmake/modules/FindVivanteSDK.cmake new file mode 100644 index 000000000..ded88850d --- /dev/null +++ b/samples/nbg_runner_pybind/cmake/modules/FindVivanteSDK.cmake @@ -0,0 +1,14 @@ +if(NOT VIVANTE_SDK_DIR) + message(FATAL_ERROR "VIVANTE_SDK_DIR is not set") +endif() + +find_library( + OPENVX_LIB + NAMES OpenVX + HINTS ${VIVANTE_SDK_DIR}/lib ${VIVANTE_SDK_DIR}/lib64 ${VIVANTE_SDK_DIR}/drivers + REQUIRED +) + +add_library(viv_sdk INTERFACE) +target_link_libraries(viv_sdk INTERFACE ${OPENVX_LIB}) +target_include_directories(viv_sdk INTERFACE ${VIVANTE_SDK_DIR}/include) diff --git a/samples/nbg_runner_pybind/examples/data/bird.jpg b/samples/nbg_runner_pybind/examples/data/bird.jpg new file mode 100644 index 000000000..a797a42b0 Binary files /dev/null and b/samples/nbg_runner_pybind/examples/data/bird.jpg differ diff --git a/samples/nbg_runner_pybind/examples/data/cat.jpg b/samples/nbg_runner_pybind/examples/data/cat.jpg new file mode 100644 index 000000000..5f681ec7e Binary files /dev/null and b/samples/nbg_runner_pybind/examples/data/cat.jpg differ diff --git a/samples/nbg_runner_pybind/examples/data/imagenet1k_labels.txt b/samples/nbg_runner_pybind/examples/data/imagenet1k_labels.txt new file mode 100644 index 000000000..fe811239d --- /dev/null +++ b/samples/nbg_runner_pybind/examples/data/imagenet1k_labels.txt @@ -0,0 +1,1001 @@ +background +tench +goldfish +great white shark +tiger shark +hammerhead +electric ray +stingray +cock +hen +ostrich +brambling +goldfinch +house finch +junco +indigo bunting +robin +bulbul +jay +magpie +chickadee +water ouzel +kite +bald eagle +vulture +great grey owl +European fire salamander +common newt +eft +spotted salamander +axolotl +bullfrog +tree frog +tailed frog +loggerhead +leatherback turtle +mud turtle +terrapin +box turtle +banded gecko +common iguana +American chameleon +whiptail +agama +frilled lizard +alligator lizard +Gila monster +green lizard +African chameleon +Komodo dragon +African crocodile +American alligator +triceratops +thunder snake +ringneck snake +hognose snake +green snake +king snake +garter snake +water snake +vine snake +night snake +boa constrictor +rock python +Indian cobra +green mamba +sea snake +horned viper +diamondback +sidewinder +trilobite +harvestman +scorpion +black and gold garden spider +barn spider +garden spider +black widow +tarantula +wolf spider +tick +centipede +black grouse +ptarmigan +ruffed grouse +prairie chicken +peacock +quail +partridge +African grey +macaw +sulphur-crested cockatoo +lorikeet +coucal +bee eater +hornbill +hummingbird +jacamar +toucan +drake +red-breasted merganser +goose +black swan +tusker +echidna +platypus +wallaby +koala +wombat +jellyfish +sea anemone +brain coral +flatworm +nematode +conch +snail +slug +sea slug +chiton +chambered nautilus +Dungeness crab +rock crab +fiddler crab +king crab +American lobster +spiny lobster +crayfish +hermit crab +isopod +white stork +black stork +spoonbill +flamingo +little blue heron +American egret +bittern +crane +limpkin +European gallinule +American coot +bustard +ruddy turnstone +red-backed sandpiper +redshank +dowitcher +oystercatcher +pelican +king penguin +albatross +grey whale +killer whale +dugong +sea lion +Chihuahua +Japanese spaniel +Maltese dog +Pekinese +Shih-Tzu +Blenheim spaniel +papillon +toy terrier +Rhodesian ridgeback +Afghan hound +basset +beagle +bloodhound +bluetick +black-and-tan coonhound +Walker hound +English foxhound +redbone +borzoi +Irish wolfhound +Italian greyhound +whippet +Ibizan hound +Norwegian elkhound +otterhound +Saluki +Scottish deerhound +Weimaraner +Staffordshire bullterrier +American Staffordshire terrier +Bedlington terrier +Border terrier +Kerry blue terrier +Irish terrier +Norfolk terrier +Norwich terrier +Yorkshire terrier +wire-haired fox terrier +Lakeland terrier +Sealyham terrier +Airedale +cairn +Australian terrier +Dandie Dinmont +Boston bull +miniature schnauzer +giant schnauzer +standard schnauzer +Scotch terrier +Tibetan terrier +silky terrier +soft-coated wheaten terrier +West Highland white terrier +Lhasa +flat-coated retriever +curly-coated retriever +golden retriever +Labrador retriever +Chesapeake Bay retriever +German short-haired pointer +vizsla +English setter +Irish setter +Gordon setter +Brittany spaniel +clumber +English springer +Welsh springer spaniel +cocker spaniel +Sussex spaniel +Irish water spaniel +kuvasz +schipperke +groenendael +malinois +briard +kelpie +komondor +Old English sheepdog +Shetland sheepdog +collie +Border collie +Bouvier des Flandres +Rottweiler +German shepherd +Doberman +miniature pinscher +Greater Swiss Mountain dog +Bernese mountain dog +Appenzeller +EntleBucher +boxer +bull mastiff +Tibetan mastiff +French bulldog +Great Dane +Saint Bernard +Eskimo dog +malamute +Siberian husky +dalmatian +affenpinscher +basenji +pug +Leonberg +Newfoundland +Great Pyrenees +Samoyed +Pomeranian +chow +keeshond +Brabancon griffon +Pembroke +Cardigan +toy poodle +miniature poodle +standard poodle +Mexican hairless +timber wolf +white wolf +red wolf +coyote +dingo +dhole +African hunting dog +hyena +red fox +kit fox +Arctic fox +grey fox +tabby +tiger cat +Persian cat +Siamese cat +Egyptian cat +cougar +lynx +leopard +snow leopard +jaguar +lion +tiger +cheetah +brown bear +American black bear +ice bear +sloth bear +mongoose +meerkat +tiger beetle +ladybug +ground beetle +long-horned beetle +leaf beetle +dung beetle +rhinoceros beetle +weevil +fly +bee +ant +grasshopper +cricket +walking stick +cockroach +mantis +cicada +leafhopper +lacewing +dragonfly +damselfly +admiral +ringlet +monarch +cabbage butterfly +sulphur butterfly +lycaenid +starfish +sea urchin +sea cucumber +wood rabbit +hare +Angora +hamster +porcupine +fox squirrel +marmot +beaver +guinea pig +sorrel +zebra +hog +wild boar +warthog +hippopotamus +ox +water buffalo +bison +ram +bighorn +ibex +hartebeest +impala +gazelle +Arabian camel +llama +weasel +mink +polecat +black-footed ferret +otter +skunk +badger +armadillo +three-toed sloth +orangutan +gorilla +chimpanzee +gibbon +siamang +guenon +patas +baboon +macaque +langur +colobus +proboscis monkey +marmoset +capuchin +howler monkey +titi +spider monkey +squirrel monkey +Madagascar cat +indri +Indian elephant +African elephant +lesser panda +giant panda +barracouta +eel +coho +rock beauty +anemone fish +sturgeon +gar +lionfish +puffer +abacus +abaya +academic gown +accordion +acoustic guitar +aircraft carrier +airliner +airship +altar +ambulance +amphibian +analog clock +apiary +apron +ashcan +assault rifle +backpack +bakery +balance beam +balloon +ballpoint +Band Aid +banjo +bannister +barbell +barber chair +barbershop +barn +barometer +barrel +barrow +baseball +basketball +bassinet +bassoon +bathing cap +bath towel +bathtub +beach wagon +beacon +beaker +bearskin +beer bottle +beer glass +bell cote +bib +bicycle-built-for-two +bikini +binder +binoculars +birdhouse +boathouse +bobsled +bolo tie +bonnet +bookcase +bookshop +bottlecap +bow +bow tie +brass +brassiere +breakwater +breastplate +broom +bucket +buckle +bulletproof vest +bullet train +butcher shop +cab +caldron +candle +cannon +canoe +can opener +cardigan +car mirror +carousel +carpenter's kit +carton +car wheel +cash machine +cassette +cassette player +castle +catamaran +CD player +cello +cellular telephone +chain +chainlink fence +chain mail +chain saw +chest +chiffonier +chime +china cabinet +Christmas stocking +church +cinema +cleaver +cliff dwelling +cloak +clog +cocktail shaker +coffee mug +coffeepot +coil +combination lock +computer keyboard +confectionery +container ship +convertible +corkscrew +cornet +cowboy boot +cowboy hat +cradle +crane +crash helmet +crate +crib +Crock Pot +croquet ball +crutch +cuirass +dam +desk +desktop computer +dial telephone +diaper +digital clock +digital watch +dining table +dishrag +dishwasher +disk brake +dock +dogsled +dome +doormat +drilling platform +drum +drumstick +dumbbell +Dutch oven +electric fan +electric guitar +electric locomotive +entertainment center +envelope +espresso maker +face powder +feather boa +file +fireboat +fire engine +fire screen +flagpole +flute +folding chair +football helmet +forklift +fountain +fountain pen +four-poster +freight car +French horn +frying pan +fur coat +garbage truck +gasmask +gas pump +goblet +go-kart +golf ball +golfcart +gondola +gong +gown +grand piano +greenhouse +grille +grocery store +guillotine +hair slide +hair spray +half track +hammer +hamper +hand blower +hand-held computer +handkerchief +hard disc +harmonica +harp +harvester +hatchet +holster +home theater +honeycomb +hook +hoopskirt +horizontal bar +horse cart +hourglass +iPod +iron +jack-o'-lantern +jean +jeep +jersey +jigsaw puzzle +jinrikisha +joystick +kimono +knee pad +knot +lab coat +ladle +lampshade +laptop +lawn mower +lens cap +letter opener +library +lifeboat +lighter +limousine +liner +lipstick +Loafer +lotion +loudspeaker +loupe +lumbermill +magnetic compass +mailbag +mailbox +maillot +maillot +manhole cover +maraca +marimba +mask +matchstick +maypole +maze +measuring cup +medicine chest +megalith +microphone +microwave +military uniform +milk can +minibus +miniskirt +minivan +missile +mitten +mixing bowl +mobile home +Model T +modem +monastery +monitor +moped +mortar +mortarboard +mosque +mosquito net +motor scooter +mountain bike +mountain tent +mouse +mousetrap +moving van +muzzle +nail +neck brace +necklace +nipple +notebook +obelisk +oboe +ocarina +odometer +oil filter +organ +oscilloscope +overskirt +oxcart +oxygen mask +packet +paddle +paddlewheel +padlock +paintbrush +pajama +palace +panpipe +paper towel +parachute +parallel bars +park bench +parking meter +passenger car +patio +pay-phone +pedestal +pencil box +pencil sharpener +perfume +Petri dish +photocopier +pick +pickelhaube +picket fence +pickup +pier +piggy bank +pill bottle +pillow +ping-pong ball +pinwheel +pirate +pitcher +plane +planetarium +plastic bag +plate rack +plow +plunger +Polaroid camera +pole +police van +poncho +pool table +pop bottle +pot +potter's wheel +power drill +prayer rug +printer +prison +projectile +projector +puck +punching bag +purse +quill +quilt +racer +racket +radiator +radio +radio telescope +rain barrel +recreational vehicle +reel +reflex camera +refrigerator +remote control +restaurant +revolver +rifle +rocking chair +rotisserie +rubber eraser +rugby ball +rule +running shoe +safe +safety pin +saltshaker +sandal +sarong +sax +scabbard +scale +school bus +schooner +scoreboard +screen +screw +screwdriver +seat belt +sewing machine +shield +shoe shop +shoji +shopping basket +shopping cart +shovel +shower cap +shower curtain +ski +ski mask +sleeping bag +slide rule +sliding door +slot +snorkel +snowmobile +snowplow +soap dispenser +soccer ball +sock +solar dish +sombrero +soup bowl +space bar +space heater +space shuttle +spatula +speedboat +spider web +spindle +sports car +spotlight +stage +steam locomotive +steel arch bridge +steel drum +stethoscope +stole +stone wall +stopwatch +stove +strainer +streetcar +stretcher +studio couch +stupa +submarine +suit +sundial +sunglass +sunglasses +sunscreen +suspension bridge +swab +sweatshirt +swimming trunks +swing +switch +syringe +table lamp +tank +tape player +teapot +teddy +television +tennis ball +thatch +theater curtain +thimble +thresher +throne +tile roof +toaster +tobacco shop +toilet seat +torch +totem pole +tow truck +toyshop +tractor +trailer truck +tray +trench coat +tricycle +trimaran +tripod +triumphal arch +trolleybus +trombone +tub +turnstile +typewriter keyboard +umbrella +unicycle +upright +vacuum +vase +vault +velvet +vending machine +vestment +viaduct +violin +volleyball +waffle iron +wall clock +wallet +wardrobe +warplane +washbasin +washer +water bottle +water jug +water tower +whiskey jug +whistle +wig +window screen +window shade +Windsor tie +wine bottle +wing +wok +wooden spoon +wool +worm fence +wreck +yawl +yurt +web site +comic book +crossword puzzle +street sign +traffic light +book jacket +menu +plate +guacamole +consomme +hot pot +trifle +ice cream +ice lolly +French loaf +bagel +pretzel +cheeseburger +hotdog +mashed potato +head cabbage +broccoli +cauliflower +zucchini +spaghetti squash +acorn squash +butternut squash +cucumber +artichoke +bell pepper +cardoon +mushroom +Granny Smith +strawberry +orange +lemon +fig +pineapple +banana +jackfruit +custard apple +pomegranate +hay +carbonara +chocolate sauce +dough +meat loaf +pizza +potpie +burrito +red wine +espresso +cup +eggnog +alp +bubble +cliff +coral reef +geyser +lakeside +promontory +sandbar +seashore +valley +volcano +ballplayer +groom +scuba diver +rapeseed +daisy +yellow lady's slipper +corn +acorn +hip +buckeye +coral fungus +agaric +gyromitra +stinkhorn +earthstar +hen-of-the-woods +bolete +ear +toilet tissue diff --git a/samples/nbg_runner_pybind/examples/demo.py b/samples/nbg_runner_pybind/examples/demo.py new file mode 100644 index 000000000..6293025a6 --- /dev/null +++ b/samples/nbg_runner_pybind/examples/demo.py @@ -0,0 +1,39 @@ +from typing import Any +from argparse import ArgumentParser +from pathlib import Path +import numpy as np + +from nbg_runner import OVXExecutor + + +def get_args() -> Any: + parser = ArgumentParser() + parser.add_argument( + "--nbg", "-m", + type=Path, + default="examples/models/conv2d_relu_maxpool2d_fp32.nbg", + help="Path to NBG file." + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + nbg_path: Path = args.nbg + + executor = OVXExecutor(nbg_path) + num_inputs = executor.get_num_inputs() + num_outputs = executor.get_num_outputs() + + input_info = executor.get_input_info(0) + output_info = executor.get_output_info(0) + + input_tensor = np.ones( + shape=input_info.shape, + dtype=input_info.dtype + ) + + executor.set_input(0, input_tensor) + executor.run() + output_tensor = executor.get_output(0) + print(output_tensor) diff --git a/samples/nbg_runner_pybind/examples/imagenet1k.py b/samples/nbg_runner_pybind/examples/imagenet1k.py new file mode 100644 index 000000000..c212db414 --- /dev/null +++ b/samples/nbg_runner_pybind/examples/imagenet1k.py @@ -0,0 +1,76 @@ +from typing import Any, List +from numpy.typing import NDArray +from argparse import ArgumentParser +from pathlib import Path +from nbg_runner import OVXExecutor +import cv2 as cv +import numpy as np + + +def get_args() -> Any: + parser = ArgumentParser() + parser.add_argument( + "--nbg", "-m", + type=Path, + required=True, + help="Path to NBG file." + ) + parser.add_argument( + "--image", "-i", + type=Path, + required=True, + help="Path to image file." + ) + parser.add_argument( + "--labels", "-l", + type=Path, + required=True, + help="Path to classification labels." + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + nbg_path: Path = args.nbg + img_path: Path = args.image + labels_path: Path = args.labels + + # Load NBG and query I/O params. + executor = OVXExecutor(nbg_path) + num_inputs = executor.get_num_inputs() + num_outputs = executor.get_num_outputs() + input_info = executor.get_input_info(index=0) + output_info = executor.get_output_info(index=0) + + input_size = input_info.shape[1:3] + num_cls = output_info.shape[1] + + # Load input image. + img_hwc: NDArray[np.uint8] = cv.imread(str(img_path), cv.IMREAD_COLOR) + hi, wi = input_size + img_hwc = cv.resize(img_hwc, dsize=(wi, hi)) + img_hwc = cv.cvtColor(img_hwc, cv.COLOR_BGR2RGB) + img_nhwc = np.expand_dims(img_hwc, axis=0) + + if input_info.dtype == "float32": + img_nhwc = img_nhwc.astype(np.float32) / np.iinfo(np.uint8).max + + # Load classification labels. + cls_labels: List[str] = [] + with open(labels_path, mode="r") as f: + for label in f: + cls_labels.append(label.strip()) + + # Run inference. + # executor.set_inputs([img_nhwc]) + executor.set_input(index=0, input_tensor=img_nhwc) + executor.run() + scores = executor.get_output(index=0) + # scores = executor.get_outputs()[0] + + cls = np.argmax(scores, axis=1) + cls = np.squeeze(cls, axis=0).item() + + cls_label = cls_labels[cls] + print(f"Classification result: {cls_label}") diff --git a/samples/nbg_runner_pybind/examples/models/conv2d_relu_maxpool2d_fp32.nbg b/samples/nbg_runner_pybind/examples/models/conv2d_relu_maxpool2d_fp32.nbg new file mode 100644 index 000000000..33dba0584 Binary files /dev/null and b/samples/nbg_runner_pybind/examples/models/conv2d_relu_maxpool2d_fp32.nbg differ diff --git a/samples/nbg_runner_pybind/python/nbg_runner/__init__.py b/samples/nbg_runner_pybind/python/nbg_runner/__init__.py new file mode 100644 index 000000000..c6374fbdc --- /dev/null +++ b/samples/nbg_runner_pybind/python/nbg_runner/__init__.py @@ -0,0 +1,6 @@ +""" VSI NBG runner python module""" + +__version__ = "0.0.1" +__author__ = "Verisilicon" + +from .vx.ovx_executor import OVXExecutor, OVXTensorInfo diff --git a/samples/nbg_runner_pybind/python/nbg_runner/_binding/__init__.py b/samples/nbg_runner_pybind/python/nbg_runner/_binding/__init__.py new file mode 100644 index 000000000..625d9c920 --- /dev/null +++ b/samples/nbg_runner_pybind/python/nbg_runner/_binding/__init__.py @@ -0,0 +1,4 @@ +from ._nbg_runner import ( + OVXExecutor, + OVXTensorInfo +) \ No newline at end of file diff --git a/samples/nbg_runner_pybind/python/nbg_runner/vx/__init__.py b/samples/nbg_runner_pybind/python/nbg_runner/vx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/samples/nbg_runner_pybind/python/nbg_runner/vx/ovx_executor.py b/samples/nbg_runner_pybind/python/nbg_runner/vx/ovx_executor.py new file mode 100644 index 000000000..fff9d5f13 --- /dev/null +++ b/samples/nbg_runner_pybind/python/nbg_runner/vx/ovx_executor.py @@ -0,0 +1,70 @@ +from typing import List, Tuple, Sequence +from numpy.typing import NDArray +import numpy as np +from pathlib import Path +from nbg_runner import _binding + + +class OVXTensorInfo: + rank: int = ... + shape: Tuple[int, ...] = ... + dtype: str = ... + qtype: str = ... + scale: float = ... + zero_point: int = ... + fixed_point_pos: int = ... + + +class OVXExecutor: + def __init__(self, nbg_path: Path) -> None: + self._exec = _binding.OVXExecutor(nbg_path) + self._exec.init() + + def get_num_inputs(self) -> int: + return self._exec.get_num_inputs() + + def get_num_outputs(self) -> int: + return self._exec.get_num_outputs() + + def get_input_info(self, index: int) -> OVXTensorInfo: + return self._exec.get_input_info(index) + + def get_output_info(self, index: int) -> OVXTensorInfo: + return self._exec.get_output_info(index) + + def get_input_infos(self) -> List[OVXTensorInfo]: + input_infos: List[OVXTensorInfo] = [] + num_inputs = self.get_num_inputs() + for i in range(num_inputs): + input_infos.append(self.get_input_info(i)) + return input_infos + + def get_output_infos(self) -> List[OVXTensorInfo]: + output_infos: List[OVXTensorInfo] = [] + num_outputs = self.get_num_outputs() + for i in range(num_outputs): + output_infos.append(self.get_output_info(i)) + return output_infos + + def set_input(self, index: int, input_tensor: NDArray) -> None: + return self._exec.set_input(index, input_tensor) + + def get_output(self, index: int) -> NDArray: + output_tensor: NDArray = self._exec.get_output(index) + return output_tensor + + def set_inputs(self, input_tensors: Sequence[NDArray]) -> None: + for i, tensor in enumerate(input_tensors): + self.set_input(i, tensor) + + def get_outputs(self) -> List[NDArray]: + output_tensors: List[NDArray] = [] + num_outputs = self.get_num_outputs() + for i in range(num_outputs): + output_tensor = self.get_output(i) + output_tensors.append(output_tensor) + + return output_tensors + + def run(self) -> None: + self._exec.run() diff --git a/samples/nbg_runner_pybind/src/CMakeLists.txt b/samples/nbg_runner_pybind/src/CMakeLists.txt new file mode 100644 index 000000000..a747dc98d --- /dev/null +++ b/samples/nbg_runner_pybind/src/CMakeLists.txt @@ -0,0 +1,9 @@ +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +pybind11_add_module(_nbg_runner ${CMAKE_CURRENT_SOURCE_DIR}/nbg_runner_pybind.cpp) + +add_subdirectory(vx) +target_link_libraries(_nbg_runner + PRIVATE + nbg_runner::vx +) diff --git a/samples/nbg_runner_pybind/src/nbg_runner_pybind.cpp b/samples/nbg_runner_pybind/src/nbg_runner_pybind.cpp new file mode 100644 index 000000000..bfd49565f --- /dev/null +++ b/samples/nbg_runner_pybind/src/nbg_runner_pybind.cpp @@ -0,0 +1,129 @@ +/**************************************************************************** +* +* Copyright (c) 2020-2024 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ + +#include +#include +#include +#include + +#include +#include +#include + +#include "vx/ovx_executor.hpp" +#include "vx/utils.hpp" + +namespace vsi::nbg_runner::python { +namespace py = pybind11; +namespace fs = std::filesystem; + +PYBIND11_MODULE(_nbg_runner, m) { + using namespace vsi::nbg_runner::vx; + + // clang-format off + py::class_(m, "OVXExecutor") + .def(py::init()) + .def(py::init([](const py::buffer& nbg_buffer) { + auto buffer_info = nbg_buffer.request(false); + return std::make_unique(reinterpret_cast(buffer_info.ptr), buffer_info.size); + })) + .def("init", &OVXExecutor::init) + .def("get_num_inputs", &OVXExecutor::get_num_inputs) + .def("get_num_outputs", &OVXExecutor::get_num_outputs) + .def("get_input_info", &OVXExecutor::get_input_info) + .def("get_output_info", &OVXExecutor::get_output_info) + .def("set_input", [](OVXExecutor* executor, size_t index, const py::buffer& buffer) { + auto buffer_info = buffer.request(false); + std::array vx_shape = {0}; + std::array vx_strides = {0}; + std::reverse_copy(buffer_info.shape.cbegin(), buffer_info.shape.cend(), vx_shape.begin()); + std::reverse_copy(buffer_info.strides.cbegin(), buffer_info.strides.cend(), vx_strides.begin()); + executor->copy_to_input( + index, + buffer_info.ptr, + buffer_info.ndim, + vx_shape.data(), + vx_strides.data() + ); + }) + .def("get_output", [](OVXExecutor* executor, size_t index) -> py::array { + auto tensor_info = executor->get_output_info(index); + auto np_dtype = py::dtype(get_vx_dtype_str(tensor_info.data_type).data()); + auto np_shape = std::vector(tensor_info.rank); + std::reverse_copy(tensor_info.shape.data(), tensor_info.shape.data() + tensor_info.rank, np_shape.begin()); + + auto np_tensor = py::array(np_dtype, np_shape); + auto buffer_info = np_tensor.request(true); + + std::array vx_strides = {0}; + std::reverse_copy(buffer_info.strides.cbegin(), buffer_info.strides.cend(), vx_strides.begin()); + + executor->copy_from_output( + index, + buffer_info.ptr, + tensor_info.rank, + tensor_info.shape.data(), + vx_strides.data() + ); + return np_tensor; + }) + .def("run", &OVXExecutor::run) + ; + + py::class_(m, "OVXTensorInfo") + .def_readonly("rank", &OVXTensorInfo::rank) + .def_property_readonly("shape", [](OVXTensorInfo* tensor_info) -> py::tuple { + size_t rank = tensor_info->rank; + auto shape_tuple = py::tuple(rank); + for (size_t i = 0; i < rank; i++){ + shape_tuple[i] = tensor_info->shape[rank - i - 1]; + } + return shape_tuple; + }) + .def_property_readonly("dtype", [](OVXTensorInfo* tensor_info) -> std::string_view { + return get_vx_dtype_str(tensor_info->data_type); + }) + .def_property_readonly("qtype", [](OVXTensorInfo* tensor_info) -> std::string_view { + return get_vx_qtype_str(tensor_info->quant_type); + }) + .def_property_readonly("scale", [](OVXTensorInfo* tensor_info) -> float { + return tensor_info->quant_type == VX_QUANT_AFFINE_SCALE + ? tensor_info->quant_param.affine.scale + : 0.0F; + }) + .def_property_readonly("zero_point", [](OVXTensorInfo* tensor_info) -> int32_t { + return tensor_info->quant_type == VX_QUANT_AFFINE_SCALE + ? tensor_info->quant_param.affine.zeroPoint + : 0; + }) + .def_property_readonly("fixed_point_pos", [](OVXTensorInfo* tensor_info) -> int8_t { + return tensor_info->quant_type == VX_QUANT_DYNAMIC_FIXED_POINT + ? tensor_info->quant_param.dfp.fixed_point_pos + : static_cast(0); + }) + ; + // clang-format on +} + +} // namespace vsi::nbg_runner::python diff --git a/samples/nbg_runner_pybind/src/vx/CMakeLists.txt b/samples/nbg_runner_pybind/src/vx/CMakeLists.txt new file mode 100644 index 000000000..8e15b2dcd --- /dev/null +++ b/samples/nbg_runner_pybind/src/vx/CMakeLists.txt @@ -0,0 +1,8 @@ +add_library(nbg_runner_vx INTERFACE) +add_library(nbg_runner::vx ALIAS nbg_runner_vx) + +target_sources(nbg_runner_vx INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/ovx_executor.cpp +) + +target_link_libraries(nbg_runner_vx INTERFACE viv_sdk) \ No newline at end of file diff --git a/samples/nbg_runner_pybind/src/vx/ovx_executor.cpp b/samples/nbg_runner_pybind/src/vx/ovx_executor.cpp new file mode 100644 index 000000000..7c503caa8 --- /dev/null +++ b/samples/nbg_runner_pybind/src/vx/ovx_executor.cpp @@ -0,0 +1,291 @@ +/**************************************************************************** +* +* Copyright (c) 2020-2024 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ + +#include "ovx_executor.hpp" + +#include + +#include +#include +#include +#include + +#include "utils.hpp" + +namespace vsi::nbg_runner::vx { + +OVXExecutor::OVXExecutor(const char* nbg_data, size_t nbg_size) { + nbg_buffer_ = std::vector(nbg_data, nbg_data + nbg_size); +} + +OVXExecutor::OVXExecutor(const fs::path& nbg_path) { + size_t nbg_size = fs::file_size(nbg_path); + auto nbg_file = std::ifstream(nbg_path, std::ios::in | std::ios::binary); + + nbg_buffer_.resize(nbg_size); + nbg_file.read(nbg_buffer_.data(), static_cast(nbg_size)); +} + +OVXExecutor::~OVXExecutor() { + for (auto& tensor : input_tensors_) { + vxReleaseTensor(&tensor); + } + for (auto& tensor : output_tensors_) { + vxReleaseTensor(&tensor); + } + + vxReleaseNode(&nbg_node_); + vxReleaseKernel(&nbg_kernel_); + vxReleaseGraph(&graph_); + vxReleaseContext(&context_); +} + +int OVXExecutor::init() { + vx_status status = VX_SUCCESS; + + context_ = vxCreateContext(); + if (context_ == nullptr) { + throw std::runtime_error("Failed to create OpenVX context."); + } + + graph_ = vxCreateGraph(context_); + status = vxGetStatus(reinterpret_cast(graph_)); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to create OpenVX graph."); + } + + nbg_kernel_ = vxImportKernelFromURL( + context_, VX_VIVANTE_IMPORT_KERNEL_FROM_POINTER, nbg_buffer_.data()); + status = vxGetStatus(reinterpret_cast(nbg_kernel_)); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to import NBG kernel."); + } + + status = query_nbg_io_infos(); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to query NBG I/O params."); + } + size_t num_inputs = input_tensors_infos_.size(); + size_t num_outputs = output_tensors_infos_.size(); + + nbg_node_ = vxCreateGenericNode(graph_, nbg_kernel_); + status = vxGetStatus(reinterpret_cast(nbg_node_)); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to create NBG node."); + } + + // Create input tensors and bind to NBG node. + for (size_t i = 0; i < num_inputs; i++) { + const auto& tensor_info = input_tensors_infos_[i]; + std::array shape; + std::transform(tensor_info.shape.cbegin(), tensor_info.shape.cend(), + shape.begin(), + [](size_t s) { return static_cast(s); }); + + vx_tensor_create_params_t tensor_create_params = { + .num_of_dims = static_cast(tensor_info.rank), + .sizes = shape.data(), + .data_format = tensor_info.data_type, + .quant_format = tensor_info.quant_type, + .quant_data = tensor_info.quant_param, + }; + vx_tensor input_tensor = vxCreateTensor2(context_, &tensor_create_params, + sizeof(tensor_create_params)); + if (input_tensor == nullptr) { + throw std::runtime_error("Failed to create input vx tensor."); + } + + vxSetParameterByIndex(nbg_node_, i, + reinterpret_cast(input_tensor)); + input_tensors_.push_back(input_tensor); + } + + // Create output tensors and bind to NBG node. + for (size_t i = 0; i < num_outputs; i++) { + const auto& tensor_info = output_tensors_infos_[i]; + std::array shape; + std::transform(tensor_info.shape.cbegin(), tensor_info.shape.cend(), + shape.begin(), + [](size_t s) { return static_cast(s); }); + + vx_tensor_create_params_t tensor_create_params = { + .num_of_dims = static_cast(tensor_info.rank), + .sizes = shape.data(), + .data_format = tensor_info.data_type, + .quant_format = tensor_info.quant_type, + .quant_data = tensor_info.quant_param, + }; + vx_tensor output_tensor = vxCreateTensor2(context_, &tensor_create_params, + sizeof(tensor_create_params)); + if (output_tensor == nullptr) { + throw std::runtime_error("Failed to create output vx tensor."); + } + + vxSetParameterByIndex(nbg_node_, num_inputs + i, + reinterpret_cast(output_tensor)); + output_tensors_.push_back(output_tensor); + } + + status = vxVerifyGraph(graph_); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to verify OpenVX graph."); + } + + return static_cast(status); +} + +int OVXExecutor::query_nbg_io_infos() { + uint32_t num_params; + vxQueryKernel(nbg_kernel_, VX_KERNEL_PARAMETERS, &num_params, + sizeof(num_params)); + + for (uint32_t i = 0; i < num_params; i++) { + vx_parameter param = vxGetKernelParameterByIndex(nbg_kernel_, i); + + vx_enum direction; + vxQueryParameter(param, VX_PARAMETER_DIRECTION, &direction, + sizeof(direction)); + + vx_meta_format meta; + vxQueryParameter(param, VX_PARAMETER_META_FORMAT, &meta, sizeof(meta)); + + OVXTensorInfo tensor_info; + vxQueryMetaFormatAttribute(meta, VX_TENSOR_NUMBER_OF_DIMS, + &tensor_info.rank, sizeof(tensor_info.rank)); + vxQueryMetaFormatAttribute(meta, VX_TENSOR_DIMS, tensor_info.shape.data(), + sizeof(tensor_info.shape)); + vxQueryMetaFormatAttribute(meta, VX_TENSOR_DATA_TYPE, + &tensor_info.data_type, + sizeof(tensor_info.data_type)); + vxQueryMetaFormatAttribute(meta, VX_TENSOR_QUANT_FORMAT, + &tensor_info.quant_type, + sizeof(tensor_info.quant_type)); + + switch (tensor_info.quant_type) { + case VX_QUANT_NONE: + break; + case VX_QUANT_AFFINE_SCALE: + vxQueryMetaFormatAttribute( + meta, VX_TENSOR_ZERO_POINT, + &tensor_info.quant_param.affine.zeroPoint, + sizeof(tensor_info.quant_param.affine.zeroPoint)); + vxQueryMetaFormatAttribute( + meta, VX_TENSOR_SCALE, &tensor_info.quant_param.affine.scale, + sizeof(tensor_info.quant_param.affine.scale)); + break; + case VX_QUANT_DYNAMIC_FIXED_POINT: + vxQueryMetaFormatAttribute( + meta, VX_TENSOR_SCALE, &tensor_info.quant_param.dfp.fixed_point_pos, + sizeof(tensor_info.quant_param.dfp.fixed_point_pos)); + break; + default: + vxReleaseParameter(¶m); + return VX_ERROR_NOT_SUPPORTED; + } + + if (direction == VX_INPUT) { + input_tensors_infos_.push_back(tensor_info); + } else if (direction == VX_OUTPUT) { + output_tensors_infos_.push_back(tensor_info); + } + + vxReleaseParameter(¶m); + } + + return VX_SUCCESS; +} + +int OVXExecutor::copy_to_input(size_t index, void* data, size_t rank, + const size_t* shape, const size_t* strides) { + if (index >= input_tensors_infos_.size()) { + throw std::out_of_range("Invalid input index."); + return VX_FAILURE; + } + + vx_tensor input_tensor = input_tensors_[index]; + auto tensor_info = input_tensors_infos_[index]; + + if (rank != tensor_info.rank) { + throw std::invalid_argument("Tensor rank mismatch."); + return VX_FAILURE; + } + + if (strides[0] != get_vx_dtype_bytes(tensor_info.data_type)) { + throw std::invalid_argument("Tensor element size mismatch."); + return VX_FAILURE; + } + + for (size_t i = 0; i < rank; i++) { + if (shape[i] != tensor_info.shape[i]) { + throw std::invalid_argument("Tensor shape mismatch."); + return VX_FAILURE; + } + } + + std::array view_start = {0}; + + vx_status status = + vxCopyTensorPatch(input_tensor, rank, view_start.data(), shape, strides, + data, VX_WRITE_ONLY, VX_MEMORY_TYPE_HOST); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to copy input data."); + return status; + } + + return VX_SUCCESS; +} + +int OVXExecutor::copy_from_output(size_t index, void* data, size_t rank, + const size_t* shape, const size_t* strides) { + if (index >= output_tensors_infos_.size()) { + throw std::runtime_error("Invalid output index."); + return VX_FAILURE; + } + + vx_tensor output_tensor = output_tensors_[index]; + + std::array view_start = {0}; + vx_status status = + vxCopyTensorPatch(output_tensor, rank, view_start.data(), shape, strides, + data, VX_READ_ONLY, VX_MEMORY_TYPE_HOST); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to copy output data."); + return status; + } + + return VX_SUCCESS; +} + +int OVXExecutor::run() { + vx_status status; + + status = vxProcessGraph(graph_); + if (status != VX_SUCCESS) { + throw std::runtime_error("Failed to run OpenVX graph."); + } + + return static_cast(status); +} + +} // namespace vsi::nbg_runner::vx diff --git a/samples/nbg_runner_pybind/src/vx/ovx_executor.hpp b/samples/nbg_runner_pybind/src/vx/ovx_executor.hpp new file mode 100644 index 000000000..a3746adae --- /dev/null +++ b/samples/nbg_runner_pybind/src/vx/ovx_executor.hpp @@ -0,0 +1,110 @@ +/**************************************************************************** +* +* Copyright (c) 2020-2024 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ + +#ifndef VSI_NBG_RUNNER_VX_OVX_EXECUTOR_HPP_ +#define VSI_NBG_RUNNER_VX_OVX_EXECUTOR_HPP_ + +#include +#include +#include +#include + +#include +#include +#include + + +namespace vsi::nbg_runner::vx { + +namespace fs = std::filesystem; + +struct OVXTensorInfo { + static constexpr size_t kMaxRank = 6; + + size_t rank; + std::array shape; + vx_enum data_type; + vx_enum quant_type; + vx_tensor_quant_param quant_param; +}; + +class OVXExecutor { + public: + explicit OVXExecutor(const char* nbg_data, size_t nbg_size); + explicit OVXExecutor(const fs::path& nbg_path); + + ~OVXExecutor(); + + /** \brief Class initialization. */ + int init(); + + /** \brief I/O params query getters. */ + [[nodiscard]] size_t get_num_inputs() const { + return input_tensors_infos_.size(); + } + [[nodiscard]] size_t get_num_outputs() const { + return output_tensors_infos_.size(); + } + [[nodiscard]] OVXTensorInfo get_input_info(size_t index) const { + return input_tensors_infos_[index]; + } + [[nodiscard]] OVXTensorInfo get_output_info(size_t index) const { + return output_tensors_infos_[index]; + } + + /** \brief Copy I/O tensor data. */ + int copy_to_input(size_t index, void* data, size_t rank, const size_t* shape, + const size_t* strides); + int copy_from_output(size_t index, void* data, size_t rank, const size_t* shape, + const size_t* strides); + + int run(); + + private: + int query_nbg_io_infos(); + + /** \brief The OpenVX context for management of all OpenVX objects. */ + vx_context context_; + /** \brief The OpenVX graph for execution. */ + vx_graph graph_; + /** \brief The OpenVX NBG node. */ + vx_node nbg_node_; + /** \brief The OpenVX NBG kernel. */ + vx_kernel nbg_kernel_; + /** \brief The OpenVX input tensors params. */ + std::vector input_tensors_infos_; + /** \brief The OpenVX output tensors params. */ + std::vector output_tensors_infos_; + /** \brief The OpenVX input tensors. */ + std::vector input_tensors_; + /** \brief The OpenVX output tensors. */ + std::vector output_tensors_; + + /** \brief The NBG buffer. */ + std::vector nbg_buffer_; +}; + +} // namespace vsi::nbg_runner::vx + +#endif \ No newline at end of file diff --git a/samples/nbg_runner_pybind/src/vx/utils.hpp b/samples/nbg_runner_pybind/src/vx/utils.hpp new file mode 100644 index 000000000..97d21bebf --- /dev/null +++ b/samples/nbg_runner_pybind/src/vx/utils.hpp @@ -0,0 +1,107 @@ +/**************************************************************************** +* +* Copyright (c) 2020-2024 Vivante Corporation +* +* Permission is hereby granted, free of charge, to any person obtaining a +* copy of this software and associated documentation files (the "Software"), +* to deal in the Software without restriction, including without limitation +* the rights to use, copy, modify, merge, publish, distribute, sublicense, +* and/or sell copies of the Software, and to permit persons to whom the +* Software is furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +* +*****************************************************************************/ + +#ifndef VSI_NBG_RUNNER_VX_UTILS_HPP_ +#define VSI_NBG_RUNNER_VX_UTILS_HPP_ + +#include + +#include + +namespace vsi::nbg_runner::vx { + +inline size_t get_vx_dtype_bytes(vx_enum data_type) { + switch (data_type) { + case VX_TYPE_INT8: + case VX_TYPE_UINT8: + case VX_TYPE_BOOL8: + case VX_TYPE_CHAR: + return 1; + case VX_TYPE_INT16: + case VX_TYPE_UINT16: + case VX_TYPE_FLOAT16: + case VX_TYPE_BFLOAT16: + return 2; + case VX_TYPE_INT32: + case VX_TYPE_UINT32: + case VX_TYPE_FLOAT32: + return 4; + case VX_TYPE_INT64: + case VX_TYPE_UINT64: + case VX_TYPE_FLOAT64: + return 8; + default: + return 0; + } +} + +inline std::string_view get_vx_dtype_str(vx_enum data_type) { + switch (data_type) { + case VX_TYPE_INT8: + return "int8"; + case VX_TYPE_UINT8: + return "uint8"; + case VX_TYPE_BOOL8: + return "bool"; + case VX_TYPE_INT16: + return "int16"; + case VX_TYPE_UINT16: + return "uint16"; + case VX_TYPE_FLOAT16: + return "float16"; + case VX_TYPE_BFLOAT16: + return "bfloat16"; + case VX_TYPE_INT32: + return "int32"; + case VX_TYPE_UINT32: + return "uint32"; + case VX_TYPE_FLOAT32: + return "float32"; + case VX_TYPE_INT64: + return "int64"; + case VX_TYPE_UINT64: + return "uint64"; + case VX_TYPE_FLOAT64: + return "float32"; + default: + return "unknown"; + } +} + +inline std::string_view get_vx_qtype_str(vx_enum quant_type) { + switch (quant_type) { + case VX_QUANT_DYNAMIC_FIXED_POINT: + return "dfp"; + case VX_QUANT_AFFINE_SCALE: + return "affine"; + case VX_QUANT_AFFINE_SCALE_PER_CHANNEL: + return "perchannel_affine"; + case VX_QUANT_NONE: + default: + return "none"; + } +} + +} // namespace vsi::nbg_runner::vx +#endif \ No newline at end of file