diff --git a/.gitignore b/.gitignore index 77d5a476cb..e3109266fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +Cargo.lock target .cargo .*.swp +*.jar diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..a612ad9813 --- /dev/null +++ b/LICENSE @@ -0,0 +1,373 @@ +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. diff --git a/README.md b/README.md index f796a7bded..0606872f25 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,42 @@ # uniffi -A very hand-wavy idea about autogenerating our FFI code and its bindings. -Don't get your hopes up just yet ;-) +This is a little experiment in building cross-platform components in rust, based on things +we've learned in the [mozilla/application-services](https://github.com/mozilla/application-services) +project. + +It's at the "very hand-wavy prototype" stage, so don't get your hopes up just yet ;-) ## What? +We're interested in building re-useable components for sync- and storage-related browser +functionality - things like [storing and syncing passwords](https://github.com/mozilla/application-services/tree/master/components/logins), +[working with bookmarks](https://github.com/mozilla/application-services/tree/master/components/places) and +[signing in to your Firefox Account](https://github.com/mozilla/application-services/tree/master/components/fxa-client). + +We want to write the code for these components once, in Rust. We want to easily re-use these components from +all the different languages and on all the different platforms for which we build browsers, which currently +includes JavaScript for PCs, Kotlin for Android, and Swift for iOS. + +And of course, we want to do this in a way that's convenient, maintainable, and difficult to mess up. + +## How? + Our current approach to building shared components in rust involves writing a lot -of boilerplate code by hand, manually flattening the rust code into an `extern "C"` -FFI layer and then manually importing and exposing that into each of Kotlin, Swift -and XPCOM+JS. The process is time-consuming, error-prone, and boring. +of boilerplate code by hand. Take the [fxa-client component](https://github.com/mozilla/application-services/tree/master/components/fxa-client) +as an example, which contains: + +* The [core functionality](https://github.com/mozilla/application-services/tree/master/components/fxa-client/src) of the component, as a Rust crate. +* A second Rust crate for [the FFI layer](https://github.com/mozilla/application-services/tree/master/components/fxa-client/ffi), + which flattens the Rust API into a set of functions and enums and opaque pointers that can be accessed from any language capable + of binding to a C-style API. +* A [Kotlin package](https://github.com/mozilla/application-services/tree/master/components/fxa-client/android/src/main/java/mozilla/appservices/fxaclient) + which wraps that C-style FFI layer back into rich classes and methods and so-on, for use in Android applications. +* A [Swift package](https://github.com/mozilla/application-services/tree/master/components/fxa-client/ios/FxAClient) + which wraps that C-style FFI layer back into rich classes and methods and so-on,for use in iOS applications. +* A third Rust crate for [exposing the core functionality to JavaScript via XPCOM](https://searchfox.org/mozilla-central/source/services/fxaccounts/rust-bridge/firefox-accounts-bridge) (which doesn't go via the C-style FFI). + +That's a lot of layers! We've developed some helpers to make it easier, but it's still a lot of repetitive +similarly-shaped code, and a lot of opportunities for human error. What if we didn't have to write all of that by hand? @@ -17,10 +45,11 @@ free using [wasm_bindgen](https://rustwasm.github.io/docs/wasm-bindgen/) and [webassembly interface types](https://hacks.mozilla.org/2019/08/webassembly-interface-types/) - imagine writing an API in Rust, annotating it with some `#[wasm_bindgen]` macros, compiling it into a webassembly bundle, and being able to import and use that bundle -from any target language! +from any target language, complete with a rich high-level API! That kind of tooling is not available to shipping applications today, but that doesn't -mean we can't take a small step in that general direction. +mean we can't take a small step in that general direction while the Rust and Wasm ecosystem +continues to evolve. ### Key Ideas @@ -40,27 +69,47 @@ but without getting too caught up in matching its precise semantics. This choice the availability of quality tooling such as the [weedle crate](https://docs.rs/weedle/0.11.0/weedle/), general familiarity around Mozilla, and the desire to avoid bikeshedding any new syntax. -We'll model the *semantics* of the component API loosely on the primitives defined by the +We'll model the *semantics* of a component's API loosely on the primitives defined by the [Wasm Interface Types](https://github.com/WebAssembly/interface-types/blob/master/proposals/interface-types/Explainer.md) proposal (henceforth "WIT"). WIT aims to solve a very similarly-shaped problem to the one we're faced with here, and by organizing this work around similar concepts, we might make it easier to one day replace all of this with direct use of WIT tooling. -In the future, we may be able to generate the Interface Definition from annotations on the rust code (in the style of `#[wasm_bindgen]`) rather than from a separate IDL file. But it's much easier to get +In the future, we may be able to generate the Interface Definition from annotations on the rust code +(in the style of `wasm_bindgen` or perhaps the [`cxx` crate](https://github.com/dtolnay/cxx)) rather than from a separate IDL file. But it's much easier to get started using a separate file. -The prototype implementation of this is in [./src/types.rs](./src/types.rs). -See [fxa-client.idl](../../fxa-client/fxa-client.idl) for an example of an interface definition. +The prototype implementation of parsing an IDL file into an in-memory representation of the component's +APIs is in [./src/types.rs](./src/types.rs). See [`arithmetic.idl`](./examples/arithmetic/arithmetic.idl) +for a simple example that actually works today, or see [`fxa-client.idl`](./examples/fxa-client/fxa-client.idl) +for an aspirational example of an interface for a real-world component. -### Primitive Types +#### Primitive Types -We'd avoid WedIDL's sparse and JS-specific types and aim to provide similar primitive types +We'll avoid WedIDL's sparse and JS-specific types and aim to provide similar primitive types to the WIT proposal: strings, bools, integers of various sizes and signedeness. We already know -how to pass these around through the FFI and the details don't seem very remarkable. +how to pass these around through a C-style FFI and the details don't seem very remarkable. + +These all pass by copying (including strings, which get copied out of Rust and into the host language +when transiting the FFI layer). + +#### Functions + +These are what they say on the tin - named callables that take typed arguments and return a typed result. +In WebIDL these always live in a namespace, like so: + +``` +namespace MyFunctions { + my_function(); + string concat(string s1, string s2); +}; +``` + +In the FFI, these are `extern "C"` functions that know how to convert values to and from Rust and the host +language. (WIT calls this "lifting" and "lowering" and we'll use the same terminology here). -They all pass by value (including strings, which get copied when transiting the FFI). -### Object Types (a.k.a. Reference Types, Handle Types, Structs, Classes, etc) +#### Object Types (a.k.a. Reference Types, Handle Types, Structs, Classes, what-have-you) These represent objects that you can instantiate, that have opaque internal state and methods that operate on that state. They're typically the "interesting" part of a component's API. We currently @@ -79,20 +128,27 @@ interface MyObject { I don't think the WIT proposal has an equivalent to these types; they're kind of like an `anyref` I guess? We should investigate further... -In the FFI, these are represented by an opaque `u64` handle. +In the FFI, instances are represented by an opaque `u64` handle, and their methods become `extern "C"` functions +that work just like plain functions, but take a handle as their first argument. -When generating component scaffolding, we could transparently create the HandleMap and the `extern "C"` functions that operate on it. We'd rely on the component code to provide a corresponding `MyObject` struct, and on Rust's typechecker to complain if the implemented methods on that struct don't match the expectations of the generated scaffolding. +When generating component scaffolding, we'll rely on hand-written Rust code to provide a `MyObject` struct with +apropriate methods. we'll transparently create a HandleMap to hold instances of this struct, and a suite of +`extern "C"` functions that load handles into struct instances and delegate to their methods. Rust's strong typing +will help us ensure that the generated scaffolding code fits together properly with the core component code. -When generating language-specific bindings, this becomes a `class` or equivalent. +When generating language-specific bindings, these becomes a `class` or equivalent. Each instance of the class +will hold a handle to the corresponding instance on the Rust side, and its methods will call the exposed +`extern "C"` functions from the FFI layer in order to delegate operations to the Rust code. TODO: * Can we use member attributes to annotate which methods require mutable vs shared access? * Can we use member attributes to identify which methods may block, and hence should be turned into a deferred/promise/whatever. -### Record Types (a.k.a. Value Types, Data Classes, Protobuf Messages, etc) +#### Record Types (a.k.a. Value Types, Data Classes, Protobuf Messages, and so-on) -These are structural types that are passed around *by value* and are typically only used -for their data. The sorts of things that we currently use JSON or Protobuf for in the FFI. +These are structural types that are passed around *by value* and are typically only used for their data. +In current hand-written components, we pass these between Rust and the host language by serializing into +JSON or Protocol Buffers and deserializing on the other side. In WebIDL this corresponds to the notion of a `dictionary`, which IMHO is not a great name for them in the context of our work here, but will do the job: @@ -104,22 +160,26 @@ dictionary MyData { } ``` -In the WIT proposal these are "records" and we use the same name here. +In the WIT proposal these are "records" and we use the same name here internally. -When generating the component scaffolding, we'd do a similar job to what's done with protobuf -today - turn the record description into a rust `struct` with appropriate fields, and helper -methods for serializing/deserializing, accessing data etc. +In the FFI layer, records *do not show up explicitly*. Functions that take or return a record will do so +via an opaque byte buffer, with the calling side serializing the record into the buffer and the receiving +side deserializing it. Buffers are always freed by the host language side (using a provided destructor +function for buffers that originate from Rust). + +When generating the component scaffolding, we'll turn the record description into a rust `struct` +with appropriate fields, and helper methods for serializing/deserializing from a byte buffer. When generating language-specific bindings, records become a "data class" or similar construct, -with field access and serialization helpers. Again, much like we currently do with protobufs. +again with field access and serialization helpers. -When passing back and forth over the FFI, records are serialized to a byte buffer and -deserialized on the other side. We could continue to use protobuf for this, but I suspect -we could come up with a simpler scheme given we control both sides of the pipe. Callers -passing a record must keep the buffer alive until the callee returns; callers receiving -a record must call a destructor to free the buffer after hydrating an object on their side. +Since we are autogenerating the code on both sides of serializing/deserializing records, we will +probably not use protocol buffers or JSON for this, but will instead use a simple bespoke encoding. +We assume that both producer and consumer will be build from the same IDL file using the same version +of `uniffi`. (Our current build tooling enforces this, and we'll try to build some simple hooks into +the generated code to ensure it as well). -### Sequences +#### Sequences Both WebIDL and WIT have a builtin `sequence` type and we should use it verbatim. @@ -129,14 +189,18 @@ interface MyObject { } ``` -We currently use ad-hoc Protobuf messages for this, e.g. the `AccountEvent` and -`AccountEvents` types in fxa-client. But it seems reasonable to support a generic -implementation on both sides of the FFI boundary. +In current hand-written compoinents we use ad-hoc Protobuf messages for this, e.g. the fxa-client +component has an `AccountEvent` record for a single event and an `AccountEvents` record for a list +of them. Since we're auto-generating things we'll instead use a more generic, re-useable implementation. + +In the FFI layer, these operate similarly to records, passing back and forth via an opque bytebuffer. -When traversing the FFI, these would be serialized into a byte buffer and parsed -back out into a Vec or Array or whatever on the other side. Just like Record types. +When generating the component scaffolding, we'll try to use Rust's rich iterator support to accept +any iterable as a sequence return value. Sequence arguments will arrive as Vecs. -### Enums +When generating language-specific bindings, sequences will show up as the native list/array/whatever type. + +#### Enums WebIDL as simple C-style enums, like this: @@ -151,42 +215,67 @@ enum AccountEventType { }; ``` -These could be encoded into an unsigned integer type for transmission over the FFI, -like the way we currently handle error numbers. +In the FFI layer these will be encoded into an unsigned integer type. + +When generating the component scaffolding, these will become a Rust enum in the obvious fashion. + +When generating language-specific bindings, these will show up however it's most obvious for an +enum to show up in that language. There is also more sophisticated stuff in there, like union types and nullable types. I haven't really thought about how to map those on to what we need. -### Callbacks +#### TODO: Nullable types + +WebIDL has support for these, and they probably have an obvious representation via Rust's +`Option` type and the equivalent in host languages. But we haven't investiated these in +any detail. + +#### TODO: Union types + +WebIDL has some support for these, and they're probably useful, but we haven't worked through +any details of how they might show up in a sensible way on both sides of the generated API. + +#### TODO: Callbacks WebIDL has some syntax for them, but I haven't looked at this in any detail at all. It seems hard, but also extremely valuable because handling callbacks across the FFI boundary has been a pain point for us in the past. -## Scaffolding Generation +## Code Generation + +Is still in its infancy, but we're working on it. The current implementation uses +[`askama`](https://docs.rs/askama/) for templating because it seems to give nice integration +with the Rust type system. + +#### Scaffolding Generation Currently a very hacky attempt in [./src/scaffolding.rs](./src/scaffolding.rs), and a `generate_component_scaffolding(idl_file: &str)` function that's intended to be used from the component's build file. -See the [fxa-client crate](../../fxa-client/build.rs) for an example of (attempting to) -use this, although it's in a very incomplete state. +#### Kotlin Bindings Generation + +Currently a very *very* hacky attempt in [./src/kotlin.rs](./src/kotlin.rs), +and it's not yet clear exactly how we should expose this for consumers. As +something done from the component's build script? As a standlone executable +that can translate an IDL file into the bindings? -It doesn't produce working Rust code, but it'll produce Rust-shaped kind-of-code -that gives you a bit of an idea what it's going for. +#### Swift Bindings Generation -Could really benefit from a templating library rather than doing a bunch of -`writeln!()` with raw source code fragements. +Totally unimplemented. If you're interested in having a go at it, try copying +the Kotlin bindings generator and adapting it to your needs! -## Language Bindings Generation +#### JS+XPCOM Bindings Generation -Currently totally unimplemented. +Totally unimplemented. If you're interested in having a go at it, try copying +the Kotlin bindings generator and adapting it to your needs! + +#### Other Host Languages + +We haven't even tried it yet! It could be a fun experiment to try to generate +some code that uses wasm-bindgen to expose a component to javascript. -A great opportunity for anyone interested to -dive in! You could try looking at the hand-written Kotlin or Swift code for -the fxa-client component, and see if you can generate something similar from -`fxa-client.idl`. Take a look at the way the scaffolding generator works to -see how to get started. ## What could possibly go wrong? @@ -202,3 +291,36 @@ APIs on every platform, and it could be harder to tweak them on a platform-by-pl basis. The resulting autogenerated code might be a lot harder to debug when things go wrong. + + +## Why didn't you just use...? + +#### WebAssembly ad wasm-bindgen + +It would be wonderful to get much or all of this for free from wasm-bindgen, but it exclusively targets +JavaScript as a host language. The upcoming Wasm Interface Types proposal should help a lot with this, +but that's still in its early stages. + +We're not aware of any production-ready WebAssembly runtimes for Android or iOS (with nice integration +with Kotlin and Swift respectively) which is a requirement for current consumers of our components. + +But aspirationally, we'd be pretty happy to one day throw away much of the code in this crate in +favour of tooling from the Wasm ecosystem. + +#### SWIG + +SWIG is a great and venerable project in this broad domain, but it's designed for C/C++ as the +implementation language rather than Rust, and at time of writing it doesn't appear to support +generating Kotlin or Swift bindings. Either of these alone might not rule it out (e.g. we could +conceivable use time spent on `uniffi` to instead write a Kotlin backgend for SWIG) but missing them +both seems to make it a bad fit for our needs. + +#### Djinni + +It targets C++ as the implementation language rather than rust, and it's been explicitly put into +"maintenance only" mode by its authors. + +#### Something else + +Please suggest it by filing an issue! If there's existing tooling to meet our needs then you might +spoil a bit of fun, but save us a whole bunch of work! diff --git a/examples/arithmetic/README.md b/examples/arithmetic/README.md new file mode 100644 index 0000000000..c5849fbcf9 --- /dev/null +++ b/examples/arithmetic/README.md @@ -0,0 +1,33 @@ +# Example uniffi component: "Arithmetic" + +This is a minimal (and very work-in-progress!) example of how to write a Rust component using +uniffi. It doesn't exercise any tricky bits of the FFI so it's a nice place to start. We have +the following so far: + +* [`./arithmetic.idl`](./arithmetic.idl), the component interface definition which exposes two + plain functions "add" and "sub". This is processed by functions in [`./build.rs`](./build.rs) + to generate Rust scaffolding for the component, and some Kotlin bindings. +* [`./src/lib.rs`](./src/lib.rs), the core implementation of the component in Rust. This basically + pulls in the generated Rust scaffolding via `include!()` and fills in function implementations. +* A tiny example program in [`./main.kt`](./main.kt) that imports the component in Kotlin, calls + one of its methods and prints the result. +* Some extremely hacky code in [`./build.rs`](./build.rs) that only works on my machine (since it + has some hard-coded file paths) that generates Kotlin bindings from the IDL, compiles them together + with `./main.kt`, and produces a runnabe `.jar` file to exercise the component. + +There is a *lot* of build and packaging detail to figure out here, but I'm able to do the following +and actually use the Rust component from Kotlin: + +* Install the kotlin command-line compiler. +* Edit `build.rs` to point it to a local copy of the JNA jar. +* Run `cargo build` in this directory; observe that it creates a file `./arithmetic.jar`. +* Try to run `./arithmetic.jar` directly using `java -jar arithmetic.jar`; observe that it fails because it can't find JNA in the classpath, and I can't figure out the right command-line flags to get it to do so. +* Unpack the jar to try running it by hand: + * `mkdir unpacked; cd unpacked` + * `unzip ../arithmetic.jar` + * `unzip -o /path/to/jna-5.2.0.jar` + * `cp ../target/debug/libuniffi_example_arithmetic.dylib ./` + * `java MainKt` + * Observe that it correctly prints the result of some simple arithmetic! + +That obviously needs to be smoother, but you get the idea :-) \ No newline at end of file diff --git a/examples/arithmetic/build.rs b/examples/arithmetic/build.rs index 95057beac6..2329be7937 100644 --- a/examples/arithmetic/build.rs +++ b/examples/arithmetic/build.rs @@ -9,6 +9,7 @@ fn main() { } fn compile_kotlin_example() { + println!("cargo:rerun-if-changed=main.kt"); let mut gen_file = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()); gen_file.push("arithmetic.kt"); // There's a whole lot of packaging and dependency-management stuff to figure out here. @@ -16,10 +17,10 @@ fn compile_kotlin_example() { let status = std::process::Command::new("kotlinc") .arg("-include-runtime") .arg("-classpath").arg("/Users/rfk/.gradle/caches/modules-2/files-2.1/net.java.dev.jna/jna/5.2.0/ed8b772eb077a9cb50e44e90899c66a9a6c00e67/jna-5.2.0.jar") - .arg("/Users/rfk/repos/mozilla/application-services/components/support/android/src/main/java/mozilla/appservices/support/native/Helpers.kt") + .arg("../../src/Helpers.kt") .arg(gen_file) - .arg("/Users/rfk/repos/mozilla/application-services/main.kt") - .arg("-d").arg("/Users/rfk/repos/mozilla/application-services/arithmetic.jar") + .arg("main.kt") + .arg("-d").arg("arithmetic.jar") .spawn() .unwrap() .wait() diff --git a/examples/arithmetic/main.kt b/examples/arithmetic/main.kt new file mode 100644 index 0000000000..a03b6e08fd --- /dev/null +++ b/examples/arithmetic/main.kt @@ -0,0 +1,5 @@ +import uniffi.example.Arithmetic; + +fun main(args: Array) { + println("2 + 3 = ${Arithmetic.add(2, 3, 1)}") +} diff --git a/examples/fxa-client/fxa-client.idl b/examples/fxa-client/fxa-client.idl new file mode 100644 index 0000000000..533044d02a --- /dev/null +++ b/examples/fxa-client/fxa-client.idl @@ -0,0 +1,134 @@ +enum DeviceCapability { + "SEND_TAB", +}; + +enum DeviceType { + "DESKTOP", + "MOBILE", +}; + +enum IncomingDeviceCommandType { + "TAB_RECEIVED", +}; + +enum AccountEventType { + "INCOMING_DEVICE_COMMAND", + "PROFILE_UPDATED", + "DEVICE_CONNECTED", + "ACCOUNT_AUTH_STATE_CHANGED", + "DEVICE_DISCONNECTED", + "ACCOUNT_DESTROYED", +}; + +dictionary ProfileInfo { + // XXX TODO: probably some of these should be `required`? + string uid; + string email; + string display_name; + string avatar; + boolean avatar_default; +}; + +dictionary AccessTokenInfo { + required string scope; + required string token; + ScopedKey key; + required uint64 expires_at; +}; + +dictionary ScopedKey { + required string kty; + required string scope; + required string k; + required string kid; +}; + +dictionary IntrospectInfo { + required boolean active = false; +}; + +dictionary DeviceInfo { + required string id; + required string display_name; + required DeviceType type; + required boolean is_current_device; + uint64 last_access_time; + PushSubscription push_subscription; + required boolean push_endpoint_expired; + sequence capabilities; +}; + +dictionary TabHistoryEntry { + required string title; + required string url; +}; + +dictionary SendTabData { + DeviceInfo from; + sequence entries; +}; + +dictionary IncomingDeviceCommand { + IncomingDeviceCommandType type; + SendTabData data; /* eventually a union type..? */ +}; + +dictionary DeviceConnectedData { + string name; +}; + +dictionary DeviceDisconnecedData { + required string id; + required boolean is_local_device; +}; + +dictionary AccountEvent { + AccountEventType type; + // No support for union types yet... + // (IncomingDeviceCommand or DeviceConnectedData or DeviceDisconnectesData) data; +}; + +dictionary MigrationState { + string blah = "blah"; +}; + +interface FirefoxAccount { + constructor(string content_url, string client_id, string redirect_uri, optional string token_server_url_override); + /*static FirefoxAccont fromJSON(string json); TODO: alternative constructors */ + string toJSON(); + + string getPairingAuthorityURL(); + string getTokenServerEndpointURL(); + string getConnectionSuccessURL(); + string getManageAccountURL(); + string getManageDevicesURL(); + + string beginOAuthFlow(sequence scopes); + string beginPairingFlow(string pairingUrl, sequence scopes); + void completeOAuthFlow(string code, string state); + void disconnect(); + + IntrospectInfo checkAuthorizationStatus(); + AccessTokenInfo getAccessToken(string scope, optional u32 ttl); + string getSessionToken(); // really whish we weren't exposing this... :-( + string getCurrentDeviceId(); + string authorizeOAuthCode(string client_id, sequence scopes, string state, optional string access_type = "online"); + void clearAccessTokenCache(); + + string copyFromSessionToken(string sessionToken, string kSync, string kXCS); // needs better return type... + string migrateFromSessionToken(string sessionToken, string kSync, string kXCS); // needs better return type... + string retryMigrateFromSessionToken(); // needs better return type... + MigrationState isInMigrationState(); + + ProfileInfo getProfile(optional boolean ignoreCache=false); + + void initializeDevice(string name, DeviceType type, sequence supportedCapabilities); + void ensureCapabilities(sequence supportedCapabilities); + void setDevicePushSubscription(string endpoint, string publicKey, string authKey); + void setDeviceDisplayName(string display_name); + sequence getDevices(optional boolean ignoreCache = false); + sequence handlePushMessage(string payload); + + sequence pollDeviceCommands(); + void sendSingleTab(string target_device_id, string title, string url); +}; \ No newline at end of file diff --git a/src/Helpers.kt b/src/Helpers.kt new file mode 100644 index 0000000000..b79dcad356 --- /dev/null +++ b/src/Helpers.kt @@ -0,0 +1,245 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package mozilla.appservices.support.native + +// TODO: We'd like to be using the a-c log tooling here, but adding that +// dependency is slightly tricky (This also could run before its log sink +// is setup!). Since logging here very much helps debugging substitution +// issues, we just use logcat. +//import android.util.Log +//import com.google.protobuf.CodedOutputStream +//import com.google.protobuf.MessageLite +import com.sun.jna.Library +import com.sun.jna.Native +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * A helper for converting a protobuf Message into a direct `java.nio.ByteBuffer` + * and its length. This avoids a copy when passing data to Rust, when compared + * to using an `Array` + */ + +/*fun T.toNioDirectBuffer(): Pair { + val len = this.serializedSize + val nioBuf = ByteBuffer.allocateDirect(len) + nioBuf.order(ByteOrder.nativeOrder()) + val output = CodedOutputStream.newInstance(nioBuf) + this.writeTo(output) + output.checkNoSpaceLeft() + return Pair(first = nioBuf, second = len) +}*/ + +sealed class MegazordError : Exception { + /** + * The name of the component we were trying to initialize when we had the error. + */ + val componentName: String + + constructor(componentName: String, msg: String) : super(msg) { + this.componentName = componentName + } + + constructor(componentName: String, msg: String, cause: Throwable) : super(msg, cause) { + this.componentName = componentName + } +} + +class IncompatibleMegazordVersion( + componentName: String, + val componentVersion: String, + val megazordLibrary: String, + val megazordVersion: String? +) : MegazordError( + componentName, + "Incompatible megazord version: library \"$componentName\" was compiled expecting " + + "app-services version \"$componentVersion\", but the megazord \"$megazordLibrary\" provides " + + "version \"${megazordVersion ?: "unknown"}\"" +) + +class MegazordNotInitialized(componentName: String) : MegazordError( + componentName, + "The application-services megazord has not yet been initialized, but is needed by \"$componentName\"" +) + +/** + * I think we'd expect this to be caused by the following two things both happening + * + * 1. Substitution not actually replacing the full megazord + * 2. Megazord initialization getting called after the first attempt to load something from the + * megazord, causing us to fall back to checking the full-megazord (and finding it, because + * of #1). + * + * It's very unlikely, but if it did happen it could be a memory safety error, so we check. + */ +class MultipleMegazordsPresent( + componentName: String, + val loadedMegazord: String, + val requestedMegazord: String +) : MegazordError( + componentName, + "Multiple megazords are present, and bindings have already been loaded from " + + "\"$loadedMegazord\" when a request to load $componentName from $requestedMegazord " + + "is made. (This probably stems from an error in your build configuration)" +) + +internal const val FULL_MEGAZORD_LIBRARY: String = "megazord" + +internal fun lookupMegazordLibrary(componentName: String, componentVersion: String): String { + return "libuniffi_example_arithmetic.dylib" + + + + val mzLibrary = System.getProperty("mozilla.appservices.megazord.library") + //Log.d("RustNativeSupport", "lib configured: ${mzLibrary ?: "none"}") + if (mzLibrary == null) { + // If it's null, then the megazord hasn't been initialized. + if (checkFullMegazord(componentName, componentVersion)) { + return FULL_MEGAZORD_LIBRARY + } + //Log.e( + // "RustNativeSupport", + // "megazord not initialized, and default not present. failing to init $componentName" + //) + throw MegazordNotInitialized(componentName) + } + + // Assume it's properly initialized if it's been initialized at all + val mzVersion = System.getProperty("mozilla.appservices.megazord.version")!! + //Log.d("RustNativeSupport", "lib version configured: $mzVersion") + + // We require exact equality, since we don't perform a major version + // bump if we change the ABI. In practice, this seems unlikely to + // cause problems, but we could come up with a scheme if this proves annoying. + if (componentVersion != mzVersion) { + //Log.e( + // "RustNativeSupport", + // "version requested by component doesn't match initialized " + + // "megazord version ($componentVersion != $mzVersion)" + //) + throw IncompatibleMegazordVersion(componentName, componentVersion, mzLibrary, mzVersion) + } + return mzLibrary +} + +/** + * Determine the megazord library name, and check that its version is + * compatible with the version of our bindings. Returns the megazord + * library name. + * + * Note: This is only public because it's called by an inline function. + * It should not be called by consumers. + */ +@Synchronized +fun findMegazordLibraryName(componentName: String, componentVersion: String): String { + //Log.d("RustNativeSupport", "findMegazordLibraryName($componentName, $componentVersion") + val mzLibraryUsed = System.getProperty("mozilla.appservices.megazord.library.used") + //Log.d("RustNativeSupport", "lib in use: ${mzLibraryUsed ?: "none"}") + val mzLibraryDetermined = lookupMegazordLibrary(componentName, componentVersion) + //Log.d("RustNativeSupport", "settled on $mzLibraryDetermined") + + // If we've already initialized the megazord, that means we've probably already loaded bindings + // from it somewhere. It would be a big problem for us to use some bindings from one lib and + // some from another, so we just fail. + if (mzLibraryUsed != null && mzLibraryDetermined != mzLibraryUsed) { + //Log.e( + // "RustNativeSupport", + // "Different than first time through ($mzLibraryDetermined != $mzLibraryUsed)!" + //) + throw MultipleMegazordsPresent(componentName, mzLibraryUsed, mzLibraryDetermined) + } + + // Mark that we're about to load bindings from the specified lib. Note that we don't do this + // in the case that the megazord check threw. + if (mzLibraryUsed != null) { + //Log.d("RustNativeSupport", "setting first time through: $mzLibraryDetermined") + System.setProperty("mozilla.appservices.megazord.library.used", mzLibraryDetermined) + } + return mzLibraryDetermined +} + +/** + * Contains all the boilerplate for loading a library binding from the megazord, + * locating it if necessary, safety-checking versions, and setting up a fallback + * if loading fails. + * + * Indirect as in, we aren't using JNA direct mapping. Eventually we'd + * like to (it's faster), but that's a problem for another day. + */ +inline fun loadIndirect( + componentName: String, + componentVersion: String +): Lib { + val mzLibrary = findMegazordLibraryName(componentName, componentVersion) + return Native.load(mzLibrary, Lib::class.java) +} + +// See the comment on full_megazord_get_version for background +// on why this exists and what we use it for. +@Suppress("FunctionNaming") +internal interface LibMegazordFfi : Library { + // Note: Rust doesn't want us to free this string (because + // it's a pain for us to arrange here), so it is actually + // correct for us to return a String over the FFI for this. + fun full_megazord_get_version(): String? +} + +/** + * Try and load the full megazord library, call the function for getting its + * version, and check it against componentVersion. + * + * - If the megazord does not exist, returns false + * - If the megazord exists and the version is valid, returns true. + * - If the megazord exists and the version is invalid, throws a IncompatibleMegazordVersion error. + * (This is done here instead of returning false so that we can provide better info in the error) + */ +internal fun checkFullMegazord(componentName: String, componentVersion: String): Boolean { + return try { + //Log.d( + // "RustNativeSupport", + // "No lib configured, trying full megazord" + //) + // It's not ideal to do this every time, but it should be rare, not too costly, + // and the workaround for the app is simple (just init the megazord). + val lib = Native.load(FULL_MEGAZORD_LIBRARY, LibMegazordFfi::class.java) + + val version = lib.full_megazord_get_version() + + //Log.d( + // "RustNativeSupport", + // "found full megazord, it self-reports version as: ${version ?: "unknown"}" + //) + if (version == null) { + throw IncompatibleMegazordVersion( + componentName, + componentVersion, + FULL_MEGAZORD_LIBRARY, + null + ) + } + + if (version != componentVersion) { + //Log.e( + // "RustNativeSupport", + // "found default megazord, but versions don't match ($version != $componentVersion)" + //) + throw IncompatibleMegazordVersion( + componentName, + componentVersion, + FULL_MEGAZORD_LIBRARY, + version + ) + } + + true + } catch (e: UnsatisfiedLinkError) { + //Log.e("RustNativeSupport", "Default megazord not found: ${e.localizedMessage}") + if (componentVersion.startsWith("0.0.1-SNAPSHOT")) { + //Log.i("RustNativeSupport", "It looks like you're using a local development build.") + //Log.i("RustNativeSupport", "You may need to check that `rust.targets` contains the appropriate platforms.") + } + false + } +} diff --git a/src/kotlin.rs b/src/kotlin.rs index c79c9a015a..4000bb9b0a 100644 --- a/src/kotlin.rs +++ b/src/kotlin.rs @@ -25,7 +25,7 @@ use super::types; // Trust me, you don't want to mess with it! // TODO: how to declare package namespace? Probably when calling the generator. -package mozilla.appservices.example; +package uniffi.example; import com.sun.jna.Library import com.sun.jna.Pointer