<?xml version="1.0"encoding="US-ASCII"?> <!-- This template is for creating an Internet Draft using xml2rfc, which is available here: http://xml.resource.org. -->encoding="UTF-8"?> <!DOCTYPE rfcSYSTEM "rfc2629.dtd"[<!-- One method to get references from the online citation libraries. There has to be one entity for each item to be referenced. An alternate method (rfc include) is described in the references. --> <!ENTITY RFC8799 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8799.xml"> <!ENTITY RFC9326 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.9326.xml"> <!ENTITY RFC2784 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.2784.xml"> <!ENTITY RFC8279 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8279.xml"> <!ENTITY RFC9322 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.9322.xml"> <!ENTITY RFC9197 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.9197.xml"> <!ENTITY RFC8926 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8926.xml"> <!ENTITY RFC7665 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7665.xml"> <!ENTITY RFC2119 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml"> <!ENTITY RFC7799 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7799.xml"> <!ENTITY RFC6830 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.6830.xml"> <!ENTITY RFC7276 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7276.xml"> <!ENTITY RFC7112 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7112.xml"> <!ENTITY RFC6833 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.6833.xml"> <!ENTITY RFC2460 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.2460.xml"> <!ENTITY RFC791 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.0791.xml"> <!ENTITY RFC6564 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.6564.xml"> <!ENTITY RFC7820 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7820.xml"> <!ENTITY RFC7821 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7821.xml"> <!ENTITY RFC8126 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8126.xml"> <!ENTITY RFC5905 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5905.xml"><!ENTITYRFC7384 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.7384.xml">nbsp " "> <!ENTITYRFC8915 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8915.xml">zwsp "​"> <!ENTITYRFC5905 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.5905.xml">nbhy "‑"> <!ENTITYRFC8039 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8039.xml"> <!ENTITY RFC8300 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8300.xml"> <!ENTITY RFC8174 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml/reference.RFC.8174.xml"> <!ENTITY I-D.ietf-ippm-ioam-data-integrity SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-ippm-ioam-data-integrity.xml"> <!ENTITY I-D.ietf-sfc-proof-of-transit SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-sfc-proof-of-transit.xml"> <!ENTITY I-D.ietf-spring-segment-routing SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-spring-segment-routing.xml"> <!ENTITY I-D.previdi-isis-segment-routing-extensions SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.previdi-isis-segment-routing-extensions.xml"> <!ENTITY I-D.ietf-ippm-6man-pdm-option SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-ippm-6man-pdm-option.xml"> <!ENTITY I-D.gont-v6ops-ipv6-ehs-in-real-world SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.gont-v6ops-ipv6-ehs-in-real-world.xml"> <!ENTITY I-D.brockners-lisp-sr SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.brockners-lisp-sr.xml"> <!ENTITY I-D.ietf-ippm-ioam-ipv6-options SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-ippm-ioam-ipv6-options.xml"> <!ENTITY I-D.ietf-6man-segment-routing-header SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-6man-segment-routing-header.xml"> <!ENTITY I-D.ietf-nvo3-vxlan-gpe SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-nvo3-vxlan-gpe.xml"> <!ENTITY I-D.ietf-ippm-ioam-conf-state SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-ippm-ioam-conf-state.xml"> <!ENTITY I-D.lapukhov-dataplane-probe SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.lapukhov-dataplane-probe.xml"> <!ENTITY I-D.ietf-ntp-packet-timestamps SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-ntp-packet-timestamps.xml"> <!ENTITY I-D.spiegel-ippm-ioam-rawexport SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.spiegel-ippm-ioam-rawexport.xml"> <!ENTITY I-D.zhou-ippm-ioam-yang SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.zhou-ippm-ioam-yang.xml"> <!ENTITY I-D.mizrahi-ippm-ioam-profile SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.mizrahi-ippm-ioam-profile.xml"> <!ENTITY I-D.weis-ippm-ioam-eth SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.weis-ippm-ioam-eth.xml"> <!ENTITY I-D.ietf-sfc-ioam-nsh SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ietf-sfc-ioam-nsh.xml"> <!ENTITY I-D.xzlnp-bier-ioam SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.xzlnp-bier-ioam.xml"> <!ENTITY I-D.brockners-ippm-ioam-geneve SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.brockners-ippm-ioam-geneve.xml"> <!ENTITY I-D.gandhi-spring-ioam-sr-mpls SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.gandhi-spring-ioam-sr-mpls.xml"> <!ENTITY I-D.ali-spring-ioam-srv6 SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.ali-spring-ioam-srv6.xml"> <!ENTITY I-D.brockners-ippm-ioam-vxlan-gpe SYSTEM "http://xml2rfc.tools.ietf.org/public/rfc/bibxml-ids/reference.I-D.brockners-ippm-ioam-vxlan-gpe.xml"> <!ENTITY AFI SYSTEM "http://www.iana.org/assignments/address-family-numbers/address-family-numbers.xml">wj "⁠"> ]><?xml-stylesheet type='text/xsl' href='rfc2629.xslt' ?> <!-- used by XSLT processors --> <!-- For a complete list and description of processing instructions (PIs), please see http://xml.resource.org/authoring/README.html. --> <!-- Below are generally applicable Processing Instructions (PIs) that most I-Ds might want to use. (Here they are set differently than their defaults in xml2rfc v1.32) --> <?rfc strict="yes" ?> <!-- give errors regarding ID-nits and DTD validation --> <!-- control the table of contents (ToC) --> <?rfc toc="yes"?> <!-- generate a ToC --> <?rfc tocdepth="4"?> <!-- the number of levels of subsections in ToC. default: 3 --> <!-- control references --> <?rfc symrefs="yes"?> <!-- use symbolic references tags, i.e, [RFC2119] instead of [1] --> <?rfc sortrefs="yes" ?> <!-- sort the reference entries alphabetically --> <!-- control vertical white space (using these PIs as follows is recommended by the RFC Editor) --> <?rfc compact="yes" ?> <!-- do not start each main section on a new page --> <?rfc subcompact="no" ?> <!-- keep one blank line between list items --> <!-- end of list of popular I-D processing instructions --><rfc xmlns:xi="http://www.w3.org/2001/XInclude" submissionType="IETF" category="info" consensus="true" docName="draft-ietf-ippm-ioam-deployment-05"ipr="trust200902"> <!-- ipr="full3978"--> <!-- category values: std, bcp, info, exp, and historic ipr values: full3667, noModification3667, noDerivatives3667 you can add the attributes updates="NNNN" and obsoletes="NNNN" they will automatically be output with "(if approved)" --> <!-- ***** FRONT MATTER ***** -->number="9378" ipr="trust200902" obsoletes="" updates="" xml:lang="en" tocInclude="true" tocDepth="4" symRefs="true" sortRefs="true" version="3"> <front><!-- The abbreviated title is used in the page header - it is only necessary if the full title is longer than 39 characters --><titleabbrev="In-situ OAM Deployment">In-situ OAMabbrev="IOAM Deployment">In Situ Operations, Administration, and Maintenance (IOAM) Deployment</title><!-- add 'role="editor"' below for the editors if appropriate --> <!-- Another author who claims to be an editor --><seriesInfo name="RFC" value="9378"/> <author fullname="Frank Brockners" initials="F." surname="Brockners" role="editor"> <organization abbrev="Cisco">Cisco Systems, Inc.</organization> <address> <postal> <extaddr>3rd Floor</extaddr> <street>Hansaallee249, 3rd Floor</street> <!-- Reorder these if your country does things differently -->249</street> <city>DUESSELDORF</city><region>NORDRHEIN-WESTFALEN</region><code>40549</code> <country>Germany</country> </postal> <email>fbrockne@cisco.com</email><!-- uri and facsimile elements may also be added --></address> </author> <author fullname="Shwetha Bhandari" initials="S." surname="Bhandari" role="editor"> <organization abbrev="Thoughtspot">Thoughtspot</organization> <address> <postal><street>3rd<extaddr>3rd Floor, IndiqubeOrion, 24th Main Rd, GardenOrion</extaddr> <extaddr>Garden Layout, HSRLayout</street> <city>Bangalore, KARNATAKA 560 102</city>Layout</extaddr> <street>24th Main Rd</street> <city>Bangalore</city> <region>KARNATAKA</region> <code>560 102</code> <country>India</country> </postal> <email>shwetha.bhandari@thoughtspot.com</email> </address> </author> <author fullname="Daniel Bernier" initials="D." surname="Bernier"> <organization abbrev="">Bell Canada</organization> <address> <postal> <street/> <city/> <code/> <country>Canada</country> </postal> <email>daniel.bernier@bell.ca</email> </address> </author> <author fullname="Tal Mizrahi" initials="T." surname="Mizrahi" role="editor"> <organization abbrev="">Huawei</organization> <address> <postal> <street>8-2 Matam</street> <city>Haifa</city> <code>3190501</code> <country>Israel</country> </postal> <email>tal.mizrahi.phd@gmail.com</email> </address> </author> <dateday="4" month="January"month="April" year="2023"/><!-- If the month and year are both specified and are the current ones, xml2rfc will fill in the current day for you. If only the current year is specified, xml2rfc will fill in the current day and month for you. If the year is not the current one, it is necessary to specify at least a month (xml2rfc assumes day="1" if not specified for the purpose of calculating the expiry date). With drafts it is normally sufficient to specify just the year. --> <!-- Meta-data Declarations --><area>tsv</area> <workgroup>ippm</workgroup><!-- WG name at the upperleft corner of the doc, IETF is fine for individual submissions. If this element is not present, the default is "Network Working Group", which is used by the RFC Editor as a nod to the history of the IETF. --> <keyword>Telemetry, Tracing,</keyword> <!-- Keywords will be incorporated into HTML output files in a meta tag but they have no effect on text or nroff output. If you submit your draft to the RFC Editor, the keywords will be used for the search engine. --><keyword>Telemetry</keyword> <keyword>Tracing</keyword> <abstract><t>In-situ<t>In situ Operations, Administration, and Maintenance (IOAM) collects operational and telemetry information in the packet while the packet traverses a path between two points in the network. This document provides a framework for IOAM deployment and provides IOAM deployment considerations and guidance.</t> </abstract> </front> <middle> <sectiontitle="Introduction" toc="default"> <t>"In-situ"toc="default" numbered="true"> <name>Introduction</name> <t>In situ Operations, Administration, and Maintenance (IOAM) collects OAM information within the packet while the packet traverses a particular network domain. The term"in-situ""in situ" refers to the fact that the OAM data is added to the data packets rather thanisbeing sent within packets specifically dedicated to OAM. IOAMis to complementcomplements mechanisms such as Ping, Traceroute, or other active probing mechanisms. In terms of "active" or "passive" OAM,"in-situ" OAMIOAM can be considered a hybrid OAM type."In-situ"In situ mechanisms do not require extra packets to be sent. IOAM adds information to the already available data packetsand thereforeand, therefore, cannot be considered passive. In terms of the classification given in <xreftarget="RFC7799"/>target="RFC7799" format="default"/>, IOAM could be portrayed as Hybrid Type I. IOAM mechanisms can be leveraged where mechanismsusingusing, e.g., ICMP do not apply or do not offer the desiredresults, such as provingresults. These situations could include:</t> <ul spacing="normal"> <li>proving that a certain traffic flow takes apre-defined path, SLApredefined path,</li> <li>verifying the Service Level Agreement (SLA) verification for the live datatraffic,traffic,</li> <li>providing detailed statistics on traffic distribution paths in networks that distribute traffic across multiple paths,oror</li> <li>providing scenarios in which probe traffic is potentially handled differently from regular data traffic by the networkdevices.</t>devices.</li> </ul> </section> <section anchor="Conventions"title="Conventions">numbered="true" toc="default"> <name>Conventions</name> <t>Abbreviations used in this document:</t><t><list hangIndent="11" style="hanging"> <t hangText="BIER:">Bit<dl newline="false" spacing="normal" indent="11"> <dt>BIER:</dt> <dd>Bit Index Explicit Replication <xreftarget="RFC8279"/></t> <t hangText="Geneve:">Generictarget="RFC8279" format="default"/></dd> <dt>Geneve:</dt> <dd>Generic Network Virtualization Encapsulation <xreftarget="RFC8926"/></t> <t hangText="GRE:">Generictarget="RFC8926" format="default"/></dd> <dt>GRE:</dt> <dd>Generic Routing Encapsulation <xreftarget="RFC2784"/></t> <t hangText="IOAM:">In-situtarget="RFC2784" format="default"/></dd> <dt>IOAM:</dt> <dd>In situ Operations, Administration, andMaintenance</t> <t hangText="MTU:">Maximum Transmit Unit</t> <t hangText="NSH:">NetworkMaintenance</dd> <dt>MTU:</dt> <dd>Maximum Transmission Unit</dd> <dt>NSH:</dt> <dd>Network Service Header <xreftarget="RFC8300"/></t> <t hangText="OAM:">Operations,target="RFC8300" format="default"/></dd> <dt>OAM:</dt> <dd>Operations, Administration, andMaintenance</t> <t hangText="POT:">Proof of Transit</t> <t hangText="VXLAN-GPE:">VirtualMaintenance</dd> <dt>POT:</dt> <dd>Proof of Transit</dd> <dt>VXLAN-GPE:</dt> <dd>Virtual eXtensible Local AreaNetwork,Network - Generic Protocol Extension <xreftarget="I-D.ietf-nvo3-vxlan-gpe"/></t> </list></t>target="I-D.ietf-nvo3-vxlan-gpe" format="default"/></dd> </dl> </section> <sectiontitle="IOAMnumbered="true" toc="default"> <name>IOAM Deployment: DomainsAnd Nodes">and Nodes</name> <t><xref target="RFC9197" format="default"/> defines the scope of IOAM as well as the different types of IOAM nodes. For improved readability, this section provides a brief overview of where IOAM applies, along with explaining the main roles of nodes that employ IOAM. Please refer to <xref target="RFC9197" format="default"/> for further details.</t> <t>IOAM is focused on "limiteddomains"domains", as defined in <xref target="RFC8799" format="default"/>. IOAM is not targeted for a deployment on the global Internet. The part of the networkwhichthat employs IOAM is referred to as the "IOAM-Domain". For example, anIOAM-domainIOAM-Domain can include an enterprise campus using physical connections between devices or an overlay network using virtual connections/or tunnels for connectivity between said devices. AnIOAM-domainIOAM-Domain is defined by its perimeter or edge. The operator of anIOAM-domainIOAM-Domain is expected to put provisions in place to ensure that packetswhichthat contain IOAM data fields do not leak beyond the edge of anIOAM domain,IOAM-Domain, e.g., usingfor examplepacket filtering methods. The operator should consider the potential operational impact of IOAMtoon mechanisms such as ECMP load-balancing schemes (e.g., load-balancing schemes based on packet length could be impacted by the increased packet size due to IOAM), path MTU (i.e., ensure that the MTU of all links within a domain is sufficiently large enough to support the increased packet size due toIOAM)IOAM), and ICMP message handling.</t> <t>An IOAM-Domain consists of "IOAM encapsulating nodes", "IOAM decapsulatingnodes"nodes", and "IOAM transit nodes". The role of a node (i.e., encapsulating, transit, decapsulating) is defined within an IOAM-Namespace (see below). A node can have different roles in different IOAM-Namespaces.</t> <t>An"IOAMIOAM encapsulatingnode"node incorporates one or moreIOAM-Option-TypesIOAM Option-Types into packets that IOAM is enabled for. If IOAM is enabled for a selected subset of the traffic, the IOAM encapsulating node is responsible for applying the IOAM functionality to the selected subset.</t> <t>An"IOAMIOAM transitnode"node updates one or more of the IOAM-Data-Fields. If both the Pre-allocated and the Incremental Trace Option-Types are present in the packet, each IOAM transit node will update at most one of these Option-Types. Note that in case both Trace Option-Types are present in a packet, it is up to the IOAM data processing systems (see <xreftarget="export"/>)target="export" format="default"/>) to integrate the data from the two Trace Option-Types to form a view of the entire journey of the packet. A transit node does not add newIOAM-Option-TypesIOAM Option-Types to apacket,packet and does not change the IOAM-Data-Fields of an IOAM Edge-to-Edge (E2E) Option-Type. </t> <t>An"IOAMIOAM decapsulatingnode"node removesIOAM-Option-Type(s)any IOAM Option-Types from packets.</t> <t>The role of anIOAM-encapsulating, IOAM-transitIOAM encapsulating, IOAM transit, orIOAM-decapsulatingIOAM decapsulating node is always performed within a specific IOAM-Namespace. This means that an IOAM nodewhich isthat is, e.g., anIOAM-decapsulatingIOAM decapsulating node for IOAM-Namespace "A" but not for IOAM-Namespace "B" will only remove theIOAM-Option-TypesIOAM Option-Types for IOAM-Namespace "A" from the packet. An IOAM decapsulating node situated at the edge of anIOAM domainIOAM-Domain removes allIOAM-Option-TypesIOAM Option-Types and associated encapsulation headers for all IOAM-Namespaces from the packet.</t> <t>IOAM-Namespaces allow for a namespace-specific definition and interpretation of IOAM-Data-Fields. Please refer to <xreftarget="ioam_namespaces"/>target="ioam_namespaces" format="default"/> for a discussion of IOAM-Namespaces.</t> <figurealign="center" anchor="IOAM-roles" title="Rolesanchor="IOAM-roles"> <name>Roles of IOAMnodes">Nodes</name> <artworkalign="left"><![CDATA[align="left" name="" type="" alt=""><![CDATA[ Export of Export of Export of Export of IOAM data IOAM data IOAM data IOAM data (optional) (optional) (optional) (optional) ^ ^ ^ ^ | | | | | | | | User +---+----+ +---+----+ +---+----+ +---+----+ packets |Encapsu-| | Transit| | Transit| |Decapsu-| -------->|lating |====>| Node |====>| Node |====>|lating |--> |Node | | A | | B | |Node | +--------+ +--------+ +--------+ +--------+ ]]></artwork> </figure> <t>IOAM nodeswhichthat add or remove the IOAM-Data-Fields can also update the IOAM-Data-Fields at the same time.OrOr, in other words, IOAM encapsulating or decapsulating nodes can also serve as IOAM transit nodes at the same time. Note that not every node in anIOAM domainIOAM-Domain needs to be an IOAM transit node. For example, a deployment might require that packets traverse a set of firewallswhichthat support IOAM. In that case, only the set of firewall nodes would be IOAM transit nodes rather than all nodes.</t> </section> <sectiontitle="Typesnumbered="true" toc="default"> <name>Types ofIOAM">IOAM</name> <t>IOAM supports different modes ofoperation, whichoperation. These modes are differentiated by the type of IOAM data fields that are being carried in the packet, the data being collected, the type of nodeswhichthat collect or updatedata as well as whetherdata, and if and how nodes export IOAM information.<list style="symbols"> <t>Per-hop tracing: OAM</t> <dl spacing="normal" newline="false"> <dt>Per-hop tracing:</dt> <dd><t>OAM information about each IOAM node a packet traverses is collected and stored within the user data packet as the packet progresses through theIOAM domain.IOAM-Domain. Potential uses of IOAM per-hop tracinginclude:<list style="symbols"> <t> Understandinclude:</t> <ul spacing="normal"> <li>Understanding the different paths that different packets traverse between an IOAM encapsulating node and an IOAM decapsulating node in a network that uses loadbalancingbalancing, such as Equal Cost Multi-Path (ECMP). This information could be used to tune the algorithm for ECMP for optimized network resourceusage.</t> <t>Operations/Troubleshooting: Understandusage.</li> <li>With regard to operations and troubleshooting, understanding which path a particular packet or set of packets take as well as what amount of jitter and delay different IOAM nodes in the path contribute to the overall delay and jitter between the IOAM encapsulating node and the IOAM decapsulatingnode.</t> </list></t> <t>Proof-of-transit: Informationnode.</li> </ul></dd> <dt>Proof of Transit:</dt> <dd>Information that a verifier node can use to verify whether a packet has traversed all nodes that it is supposed to traverse is stored within the user data packet.Proof-of-transitFor example, Proof of Transit couldfor examplebe used to verify that a packet indeed passes through all services of a service function chain (e.g., verify whether a packet indeed traversed the set of firewalls that it is expected totraverse),traverse) or whether a packet indeed took the expectedpath.</t> <t>Edge-to-edge: OAMpath.</dd> <dt>Edge-to-Edge (E2E):</dt> <dd>OAM informationwhichthat is specific to the edges of anIOAM domainIOAM-Domain is collected and stored within the user data packet.Edge-to-EdgeE2E OAM could be used to gather operational information about a particular network domain, such as the delay and jitter incurred by that network domain or the traffic matrix of the networkdomain.</t> <t>Direct export: OAMdomain.</dd> <dt>Direct Export:</dt> <dd>OAM information about each IOAM node a packet traverses is collected and immediately exported to a collector. DirectexportExport could be used in situations where per-hop tracing information is desired, but gathering the information within the packet--- as with per-hop tracing--- is not feasible. Rather than automatically correlating the per-hop tracing information, as done with per-hop tracing,direct exportDirect Export requires a collector to correlate the information from the individual nodes. In addition, all nodes enabled fordirect exportDirect Export need to be capableto exportof exporting the IOAM information to thecollector.</t> </list></t>collector.</dd> </dl> <section anchor="IOAM-tracing"title="Per-hopnumbered="true" toc="default"> <name>Per-Hop TracingIOAM">IOAM</name> <t>"IOAM tracing data" is expected to be collected at every IOAM transit node that a packet traverses to ensure visibility into the entire path that a packet takes within an IOAM-Domain.I.e.,In other words, in a typicaldeploymentdeployment, all nodes in an IOAM-Domain would participate in IOAMand thusand, thus, be IOAM transit nodes, IOAM encapsulating nodes, or IOAM decapsulating nodes. If not all nodes within a domain are IOAM capable, IOAM tracing information (i.e., node data, see below) will only be collected on those nodeswhichthat are IOAM capable. Nodeswhichthat are not IOAM capable will forward the packet without any changes to the IOAM-Data-Fields. The maximum number of hops and the minimum path MTU of theIOAM domain isIOAM-Domain are assumed to be known.</t> <t>IOAM offers two differenttrace Option-Types,Trace Option-Types: the"incremental""Incremental" Trace Option-Typeas well asand the"pre-allocated""Pre-allocated" Trace Option-Type. For a discussion about which of the two option types is the most suitable for an implementation and/or deployment, see <xreftarget="IOAM-trace-options"/>.</t>target="IOAM-trace-options" format="default"/>.</t> <t>Every node data entry holds information for a particular IOAM transit node that is traversed by a packet. The IOAM decapsulating node removesthe IOAM-Option-Type(s)any IOAM Option-Types and processes and/or exports the associated data. All IOAM-Data-Fields are defined in the context of an IOAM-Namespace.</t> <t>IOAM tracingcancan, forexampleexample, collect the following types of information:</t><t><list style="symbols"> <t>Identification<ul spacing="normal"> <li>Identification of the IOAM node. An IOAM node identifier can match to a device identifier or a particular control point or subsystem within a device.</t> <t>Identification</li> <li>Identification of the interface that a packet was received on,i.e.i.e., ingressinterface.</t> <t>Identificationinterface.</li> <li>Identification of the interface that a packet was sent out on,i.e.i.e., egressinterface.</t> <t>Timeinterface.</li> <li>Time of day when the packet was processed by the node as well as the transit delay. Different definitions of processing time are feasible and expected, though it is important that all devices of anin-situ OAM domainIOAM-Domain follow the samedefinition.</t> <t>Generic data: Format-free informationdefinition.</li> <li>Generic data, which is format-free information, where the syntax andsemanticsemantics of the informationisare defined by the operator in a specific deployment. For a specific IOAM-Namespace, all IOAM nodes should interpret the generic data the same way. Examples for generic IOAM data include geolocation information (location of the node at the time the packet was processed), buffer queue fill level or cache fill level at the time the packet was processed, or even a battery chargelevel.</t> <t>Informationlevel.</li> <li>Information to detect whether IOAM trace data was added at every hop or whether certain hops in the domain weren't IOAM transitnodes.</t> <t>Datanodes.</li> <li>Data that relates to how the packet traversed a node (transit delay, buffer occupancy in case the packet was buffered, and queue depth in case the packet wasqueued)</t> </list>The Option-Types of incremental tracingqueued).</li> </ul> <t>The Incremental Trace Option-Type andpre-allocated tracingPre-allocated Trace Option-Type are defined in <xreftarget="RFC9197"/>.</t>target="RFC9197" format="default"/>.</t> </section> <section anchor="IOAM-proof-of-transit"title="Proofnumbered="true" toc="default"> <name>Proof of TransitIOAM"> <t>IOAMIOAM</name> <t>The IOAM Proof of Transit Option-Type is to support path or service function chain <xreftarget="RFC7665"/>target="RFC7665" format="default"/> verification use cases.Proof-of-transitProof of transit could use methods like nested hashing or nested encryption of the IOAM data.</t> <t>The IOAM Proof of Transit Option-Typeconsistconsists of afixed sizefixed-size "IOAMproofProof oftransit optionTransit Option header" and "IOAMproofProof oftransit optionTransit Option data fields". Fordetailsdetails, see <xreftarget="RFC9197"/>.</t>target="RFC9197" format="default"/>.</t> </section> <section anchor="IOAM-edge-to-edge"title="Edge to Edge IOAM">numbered="true" toc="default"> <name>E2E IOAM</name> <t>The IOAMEdge-to-EdgeE2E Option-Type is to carry the data that is added by the IOAM encapsulating node and interpreted by IOAM decapsulating node. The IOAM transit nodes may process the data but must not modify it.</t> <t>The IOAMEdge-to-EdgeE2E Option-Typeconsistconsists of afixed sizefixed-size "IOAM Edge-to-Edge Option-Type header" and "IOAM Edge-to-Edge Option-Type data fields". Fordetailsdetails, see <xreftarget="RFC9197"/>.</t>target="RFC9197" format="default"/>.</t> </section> <sectiontitle="Directnumbered="true" toc="default"> <name>Direct ExportIOAM">IOAM</name> <t>Direct Export is an IOAM mode of operation within which IOAM data are to be directly exported to a collector rather thanbeingbe collected within the data packets. The IOAM Direct Export Option-Typeconsistconsists of afixed sizefixed-size "IOAM direct export option header". Direct Export for IOAM is defined in <xreftarget="RFC9326"/>.</t>target="RFC9326" format="default"/>.</t> </section> </section> <section anchor="ioam-encap"title="IOAM Encapsulations">numbered="true" toc="default"> <name>IOAM Encapsulations</name> <t>IOAM data fields and associated data types forin-situ OAMIOAM are defined in <xreftarget="RFC9197"/>.target="RFC9197" format="default"/>. Thein-situ OAMIOAM data field can be transported by a variety of transport protocols, including NSH, Segment Routing, Geneve, BIER, IPv6, etc.</t> <sectiontitle="IPv6">numbered="true" toc="default"> <name>IPv6</name> <t>IOAM encapsulation for IPv6 is defined in <xreftarget="I-D.ietf-ippm-ioam-ipv6-options"/>,target="I-D.ietf-ippm-ioam-ipv6-options" format="default"/>, which alsodiscusseddiscusses IOAM deployment considerations for IPv6networks</t>networks.</t> </section> <sectiontitle="NSH">numbered="true" toc="default"> <name>NSH</name> <t>IOAM encapsulation for NSH is defined in <xreftarget="I-D.ietf-sfc-ioam-nsh"/>.</t>target="I-D.ietf-sfc-ioam-nsh" format="default"/>.</t> </section> <sectiontitle="BIER">numbered="true" toc="default"> <name>BIER</name> <t>IOAM encapsulation for BIER is defined in <xreftarget="I-D.xzlnp-bier-ioam"/>.</t>target="I-D.xzlnp-bier-ioam" format="default"/>.</t> </section> <sectiontitle="GRE">numbered="true" toc="default"> <name>GRE</name> <t>IOAM encapsulation for GRE is outlined as part of the "EtherType Protocol Identification of In-situ OAM Data" in <xreftarget="I-D.weis-ippm-ioam-eth"/>.</t>target="I-D.weis-ippm-ioam-eth" format="default"/>.</t> </section> <sectiontitle="Geneve">numbered="true" toc="default"> <name>Geneve</name> <t>IOAM encapsulation for Geneve is defined in <xreftarget="I-D.brockners-ippm-ioam-geneve"/>.</t>target="I-D.brockners-ippm-ioam-geneve" format="default"/>.</t> </section> <sectiontitle="Segment Routing">numbered="true" toc="default"> <name>Segment Routing</name> <t>IOAM encapsulation for Segment Routing is defined in <xreftarget="I-D.gandhi-spring-ioam-sr-mpls"/>.</t>target="I-D.gandhi-mpls-ioam" format="default"/>.</t> </section> <sectiontitle="Segmentnumbered="true" toc="default"> <name>Segment Routing forIPv6">IPv6</name> <t>IOAM encapsulation for Segment Routing over IPv6 is defined in <xreftarget="I-D.ali-spring-ioam-srv6"/>.</t>target="I-D.ali-spring-ioam-srv6" format="default"/>.</t> </section> <sectiontitle="VXLAN-GPE">numbered="true" toc="default"> <name>VXLAN-GPE</name> <t>IOAM encapsulation for VXLAN-GPE is defined in <xreftarget="I-D.brockners-ippm-ioam-vxlan-gpe"/>.</t>target="I-D.brockners-ippm-ioam-vxlan-gpe" format="default"/>.</t> </section> </section> <section anchor="export"title="IOAMnumbered="true" toc="default"> <name>IOAM DataExport">Export</name> <t>IOAM nodes collect information for packets traversing a domain that supports IOAM. IOAM decapsulatingnodesnodes, as well as IOAM transitnodesnodes, can choose to retrieve IOAM information from the packet, process the informationfurtherfurther, and export the informationusingusing, e.g.,IPFIX.</t>IP Flow Information Export (IPFIX).</t> <t>Raw data export of IOAM data using IPFIX is discussed in <xreftarget="I-D.spiegel-ippm-ioam-rawexport"/>.target="I-D.spiegel-ippm-ioam-rawexport" format="default"/>. "Raw export of IOAM data" refers to a mode of operation where a node exports the IOAM data as it is received in the packet. The exporting nodeneither interprets, aggregates nor reformatsdoes not interpret, aggregate, or reformat the IOAM data before it is exported. Raw export of IOAM data is to support an operational model where the processing and interpretation of IOAM data is decoupled from the operation of encapsulating/updating/decapsulating IOAM data, which is also referred to asIOAM"IOAM data-planeoperation. The figure belowoperation". <xref target="export-arch"/> shows the separation of concerns for IOAMexport:export. Exporting IOAM data is performed by the "IOAMnode"node", which performs IOAM data-plane operation, whereas the interpretation of IOAM data is performed by one or several IOAM data processing systems. The separation of concerns is tooff-loadoffload interpretation,aggregationaggregation, and formatting of IOAM data from the nodewhichthat performs data-plane operations. In other words, a nodewhichthat is focused on data-plane operations,i.e.i.e., forwarding of packets and handling IOAMdatadata, will not be tasked to also interpret the IOAMdata, butdata. Instead, that node can leave this task to another system or a set of systems. For scalability reasons, a single IOAM node could choose to export IOAM data to several systems that process IOAMdata processing systems.data. Similarly,thereseveral monitoring systems or analytics systems can be used to further process the data received from the IOAM preprocessing systems. <xreftarget="export-arch"/>target="export-arch" format="default"/> shows an overview of IOAM export, including IOAM data processing systems andmonitoring/analyticsmonitoring and analytics systems.</t> <figurealign="center" anchor="export-arch" title="IOAM frameworkanchor="export-arch"> <name>IOAM Framework withdata export">Data Export</name> <artworkalign="left"><![CDATA[align="left" name="" type="" alt=""><![CDATA[ +--------------+ +-------------+ | | Monitoring/ | | | Analytics | | | system(s) |-+ +-------------+ ^ | Processed/interpreted/ | aggregated IOAM data | +--------------+ +-------------+ | | IOAM data | | | processing | | | system(s) |-+ +-------------+ ^ | Raw export of | IOAM data | +--------------+-------+------+--------------+ | | | | | | | | User +---+----+ +---+----+ +---+----+ +---+----+ packets |Encapsu-| | Transit| | Transit| |Decapsu-| -------->|lating |====>| Node |====>| Node |====>|lating |--> |Node | | A | | B | |Node | +--------+ +--------+ +--------+ +--------+ ]]></artwork> </figure> </section> <section anchor="IOAM-framework"title="IOAMnumbered="true" toc="default"> <name>IOAM DeploymentConsiderations">Considerations</name> <t>This section describes several concepts ofIOAM,IOAM and provides considerations that need to be takentointo account when implementing IOAM in a network domain. This includes concepts likeIOAM Namespaces,IOAM-Namespaces, IOAM Layering, traffic-sets that IOAM is appliedtoto, and IOAMloopback mode.Loopback. For a definition ofIOAM NamespacesIOAM-Namespaces and IOAMlayering,Layering, please refer to <xreftarget="RFC9197"/>.target="RFC9197" format="default"/>. IOAMloopback modeLoopback is defined in <xreftarget="RFC9322"/>.</t>target="RFC9322" format="default"/>.</t> <section anchor="ioam_namespaces"title="IOAM Namespaces">numbered="true" toc="default"> <name>IOAM-Namespaces</name> <t>IOAM-Namespaces add further context toIOAM-Option-TypesIOAM Option-Types and associated IOAM-Data-Fields. IOAM-Namespaces are defined inSection 4.3 of<xreftarget="RFC9197"/>.target="RFC9197" sectionFormat="of" section="4.3"/>. The Namespace-ID is part of the IOAM Option-Typedefinition, see e.g., Section 4.4 ofdefinition. See <xreftarget="RFC9197"/>target="RFC9197" sectionFormat="of" section="4.4"/> for IOAM Trace Option-Types orSection 4.6 of<xreftarget="RFC9197"/>target="RFC9197" sectionFormat="of" section="4.6"/> for the IOAMEdge-to-EdgeE2E Option-Type. IOAM-Namespaces support several uses:</t><t><list style="symbols"> <t>IOAM-Namespaces<ul spacing="normal"> <li>IOAM-Namespaces can be used by an operator to distinguish between different operational domains. Devices at domain edges can filter on Namespace-IDs to provide for proper IOAM-Domainisolation.</t> <t>IOAM-Namespacesisolation.</li> <li>IOAM-Namespaces provide additional context forIOAM-Data-Fields and thusIOAM-Data-Fields; thus, they ensure that IOAM-Data-Fields are unique and can be interpreted properly by management stations or network controllers. While, for example, the node identifier field does not need to be unique in a deployment (e.g., an operator may wish to use different node identifiers for different IOAM layers, even within the same device; or node identifiers might not be unique for other organizational reasons, such as after a merger of two formerly separated organizations), the combination of node_id and Namespace-ID should always be unique. Similarly, IOAM-Namespaces can be used to define how certain IOAM-Data-Fields areinterpreted:interpreted. IOAM offers three different timestamp format options. The Namespace-ID can be used to determine the timestamp format. IOAM-Data-Fields (e.g., buffer occupancy)whichthat do not have a unit associated are to be interpreted within the context ofaan IOAM-Namespace. The Namespace-ID could also be used to distinguish between different types ofinterfaces:interfaces. An interface-idcouldcould, forexampleexample, point to a physical interface (e.g., to understand which physical interface of an aggregated link is used when receiving or transmitting apacket) whereaspacket). Whereas, in anothercase itcase, an interface-id could refer to a logical interface (e.g., in case of tunnels). Namespace-IDs could be used to distinguish between the different types of interfaces.</t></li> <li> <t>IOAM-Namespaces can be used to identify different sets of devices (e.g., different types of devices) in adeployment:deployment. If an operator desires to insert different IOAM-Data-Fields based on the device, the devices could be grouped into multiple IOAM-Namespaces. This could be due to the fact that the IOAM feature set differs between different sets of devices, or it could be for reasons of optimized space usage in the packet header. It could also stem from hardware or operational limitations on the size of the trace data that can be added and processed, preventing collection of a full trace for aflow.<list style="symbols"> <t>Assigningflow.</t> <ul spacing="normal"> <li>Assigning different IOAM Namespace-IDs to different sets of nodes or network partitions and using the Namespace-ID as a selector at the IOAM encapsulating node, a full trace for a flow could be collected and constructed via partial traces in different packets of the same flow.Example: AnFor example, an operator could choose to group the devices of a domain into twoIOAM-Namespaces,IOAM-Namespaces in a waythat atthat, on average, only every second hop would be recorded by any device. To retrieve a full view of the deployment, the captured IOAM-Data-Fields of the two IOAM-Namespaces need to becorrelated.</t> <t>Assigningcorrelated.</li> <li>Assigning different IOAM Namespace-IDs to different sets of nodes or network partitions and using a separate instance of anIOAM-Option-TypeIOAM Option-Type for each Namespace-ID, a full trace for a flow could be collected and constructed via partial traces from eachIOAM-Option-TypeIOAM Option-Type in each of the packets in the flow.Example: AnFor example, an operator could choose to group the devices of a domain into twoIOAM-Namespaces,IOAM-Namespaces in a way that each IOAM-Namespace is represented by one of twoIOAM-Option-TypesIOAM Option-Types in the packet. Each node would record data only for the IOAM-Namespace that it belongs to, ignoring the otherIOAM-Option-TypeIOAM Option-Type withaan IOAM-Namespaceto whichit doesn'tbelong.belong to. To retrieve a full view of the deployment, the captured IOAM-Data-Fields of the two IOAM-Namespaces need to becorrelated.</t> </list></t> </list></t>correlated.</li> </ul> </li> </ul> </section> <sectiontitle="IOAM Layering">numbered="true" toc="default"> <name>IOAM Layering</name> <t>If several encapsulation protocols (e.g., in case of tunneling) are stacked on top of each other, IOAM-Data-Fields could be present in different protocol fields at different layers. Layering allows operators to instrument the protocol layer they want to measure. The behavior follows the ships-in-the-night model, i.e., IOAM-Data-Fields in one layer are independent of IOAM-Data-Fields in another layer. Or in otherwords: Evenwords, even though the term "layering" often implies there is some form of hierarchy and relationship, in IOAM, layers are independent of each other and don't assume any relationship among them. The different layers could, but do not havetoto, share the same IOAM encapsulation mechanisms. Similarly, the semantics of theIOAM-Data- Fields,IOAM-Data-Fields can, but do not havetoto, be associated to cross different layers. For example, a nodewhichthat inserts node-id information into two different layers could use "node-id=10" for one layer and "node-id=1000" for the second layer.</t> <t><xreftarget="IOAM-layering"/>target="IOAM-layering" format="default"/> shows an example of IOAMlayering.Layering. The figure shows a Geneve tunnel carried overIPv6IPv6, which starts at node A and ends at node D. IOAM information is encapsulated in IPv6 as well as in Geneve. At the IPv6 layer, node A is the IOAM encapsulating node (into IPv6), node D is the IOAM decapsulatingnodenode, andnodenodes B andnodeC are IOAM transit nodes. At the Geneve layer, node A is the IOAM encapsulating node (intoGeneve)Geneve), and node D is the IOAM decapsulating node (from Geneve). The use of IOAM at bothlayerslayers, as shown in the exampleherehere, could be used to reveal which nodes of an underlay (here the IPv6 network) are traversed by a tunneled packet in an overlay (here the Geneve network)--- which assumes that the IOAM information encapsulated by nodes A and D into Geneve and IPv6 is associated to each other.</t> <figurealign="center" anchor="IOAM-layering" title="IOAM layering example">anchor="IOAM-layering"> <name>IOAM Layering Example</name> <artworkalign="left"><![CDATA[align="left" name="" type="" alt=""><![CDATA[ +---+----+ +---+----+ User |Geneve | |Geneve | packets |Encapsu-| |Decapsu-| -------->|lating |==================================>|lating |--> | Node | | Node | | A | | D | +--------+ +--------+ ^ ^ | | v v +--------+ +--------+ +--------+ +--------+ |IPv6 | | IPv6 | | IPv6 | |IPv6 | |Encapsu-| | Transit| | Transit| |Decapsu-| |lating |====>| Node |====>| Node |====>|lating | | Node | | | | | | Node | | A | | B | | C | | D | +--------+ +--------+ +--------+ +--------+ ]]></artwork> </figure> </section> <section anchor="IOAM-trace-options"title="IOAMnumbered="true" toc="default"> <name>IOAM TraceOption Types">Option-Types</name> <t>IOAM offers two different IOAM Option-Types for tracing: "Incremental"Trace-Option-TypeTrace Option-Type and "Pre-allocated"Trace-Option-Type.Trace Option-Type. "Incremental" refers to a mode of operation where the packet is expanded at every IOAM node that adds IOAM-Data-Fields."Pre-Allocated""Pre-allocated" describes a mode of operation where the IOAM encapsulating node allocates room for all IOAM-Data-Fields in the entireIOAM domain.IOAM-Domain. More specifically:</t><t><list style="hanging"> <t hangText="Pre-allocated Trace-Option:">This<dl newline="false" spacing="normal"> <dt>Pre-allocated Trace Option:</dt> <dd>This trace option is defined as a container of node data fields with pre-allocated space for each node to populate its information. This option is useful for implementations where it is efficient to allocate the space once and index into the array to populate the data during transit (e.g., software forwarders often fall into thisclass).</t> <t hangText="Incremental Trace-Option:">Thisclass).</dd> <dt>Incremental Trace Option:</dt> <dd>This trace option is defined as a container of node data fields where each node allocates and pushes its node data immediately following the optionheader.</t> </list> Whichheader.</dd> </dl> <t>Which IOAMTrace-Option-TypesTrace Option-Types can be supported is not only a function of operator-definedconfiguration,configuration but may also be limited by protocol constraints unique to a given encapsulating protocol. For encapsulating protocolswhichthat support both IOAMTrace-Option-Types,Trace Option-Types, the operatordecidesdecides, by means ofconfigurationconfiguration, whichTrace-Option-Type(s)Trace Option-Type(s) will be used for a particular domain. In this case, deployments can mix deviceswhichthat include either the IncrementalTrace-Option-TypeTrace Option-Type or the Pre-allocatedTrace-Option-Type. If for exampleTrace Option-Type. For example, if different types of packet forwarders and associated different types of IOAM implementations exist in a deployment and the encapsulating protocol supports both IOAMTrace-Option-Types,Trace Option-Types, a deployment can mix deviceswhichthat include either the IncrementalTrace-Option-TypeTrace Option-Type or the Pre-allocatedTrace-Option-Type.Trace Option-Type. As a result, both Option-Types can be present in a packet. IOAM decapsulating nodes remove both types ofTrace-Option-TypesTrace Option-Types from the packet.</t> <t>The two different Option-Types cater to differentpacket forwardingpacket-forwarding infrastructures andare toallow an optimized implementation of IOAMtracing:<list style="hanging"> <t hangText="Pre-allocated Trace-Option:">Fortracing:</t> <dl newline="false" spacing="normal"> <dt>Pre-allocated Trace Option:</dt> <dd>For some implementations of packetforwardersforwarders, it is efficient to allocate the space for the maximum number of nodes that IOAM Trace Data-Fields should be collected from and insert/update information in the packet at flexiblelocations,locations based on a pointer retrieved from a field in the packet. The IOAM encapsulating node allocates an array of the size of the maximum number of nodes that IOAM Trace Data-Fields should be collected from. IOAM transit nodes index into the array to populate the data during transit. Software forwarders often fall into this class ofpacket forwarderpacket-forwarder implementations. The maximum number of nodes that IOAM information could be collected from is configured by the operator on the IOAM encapsulating node. The operator has to ensure that the packet with the pre-allocated array that carries the IOAM Data-Fields does not exceed the MTU of any of the links in theIOAM domain.</t> <t hangText="Incremental Trace-Option:">LookingIOAM-Domain.</dd> <dt>Incremental Trace Option:</dt> <dd>Looking up a pointer contained in the packet and inserting/updating information at a flexible location in the packet as a result of the pointer lookup is costly for some forwarding infrastructures. Hardware-basedpacket forwardingpacket-forwarding infrastructures often fall into this category. Consequently, hardware-based packet forwarders could choose to support theincremental IOAM-Trace-Option-Type.IOAM Incremental Trace Option-Type. Theincremental IOAM-Trace-Option-TypeIOAM Incremental Trace Option-Type eliminates the need for the IOAM transit nodes to read the full array in theTrace-Option-TypeTrace Option-Type and allows packets to grow to the size of the MTU of theIOAM domain.IOAM-Domain. IOAM transit nodes will expand the packet and insert the IOAM-Data-Fields as long as there is space available in the packet,i.e.i.e., as long as the size of the packet stays within the bounds of the MTU of the links in theIOAM domain.IOAM-Domain. There is no need for the operator to configure the IOAM encapsulation node with the maximum number of nodes that IOAM information could be collected from. The operator has to ensure that the minimum MTU of the links in theIOAM domainIOAM-Domain is known to all IOAM transitnodes.</t> </list></t>nodes.</dd> </dl> </section> <sectiontitle="Traffic-sets thatnumbered="true" toc="default"> <name>Traffic-Sets That IOAM Is AppliedTo">To</name> <t>IOAM can be deployed on all or only on subsets of the live user traffic, e.g., per interface, based on an access control list or flow specification defining a specific set of traffic, etc.</t> </section> <sectiontitle="IOAM Loopback Mode">numbered="true" toc="default"> <name>Loopback Flag</name> <t>IOAM Loopback is used to trigger each transit device along the path of a packet to send a copy of the data packet back to the source. Loopback allows an IOAM encapsulating node to trace the path to a givendestination,destination and to receive per-hop data about both the forward and the return path. Loopback is enabled by the encapsulating node setting theloopbackLoopback flag. Looped-back packets use the source address of the original packet as a destination address and the address of the nodewhichthat performs theloopbackLoopback operation as source address. Nodeswhichthat loop back a packet clear theloopbackLoopback flag before sending the copy back towards the source. Loopack applies to IOAM deployments where the encapsulating node is either a host or the start of atunnel:tunnel. For details on IOAMloopback,Loopback, please refer to <xreftarget="RFC9322"/>.</t>target="RFC9322" format="default"/>.</t> </section> <sectiontitle="IOAM Active Mode">numbered="true" toc="default"> <name>Active Flag</name> <t>TheIOAM active modeActive flag indicates that a packet is an active OAM packet as opposed to regular user data traffic. Activemodeflag is expected to be used for active measurement using IOAM. For details onIOAM active mode,the Active flag, please refer to <xreftarget="RFC9322"/>.</t>target="RFC9322" format="default"/>.</t> <t>Exampleuse-casesuse cases forIOAMthe ActiveMode include:<list style="symbols"> <t>Endpointflag include:</t> <dl spacing="normal" newline="false"> <dt>Endpoint detailed activemeasurement: Syntheticmeasurement:</dt> <dd>Synthetic probe packets are sent between the source and destination. These probe packets include a Trace Option-Type (i.e., either incremental or pre-allocated). Since the probe packets are sent between the endpoints, these packets are treated as data packets by theIOAM domain,IOAM-Domain and do not require special treatment at the IOAM layer. The source, which is also the IOAM encapsulatingnodenode, can choose to set the Active flag, providing an explicit indication that these probe packets are meant for telemetrycollection.</t> <t>IOAMcollection.</dd> <dt>IOAM active measurement using probepackets: Probepackets:</dt> <dd>Probe packets are generated and transmitted by an IOAM encapsulating node towards a destinationwhichthat is also the IOAM decapsulating node. Probe packets include a Trace Option-Type (i.e., either incremental or pre-allocated)whichthat has its Active flagset.</t> <t>IOAMset.</dd> <dt>IOAM active measurement using replicated datapackets: Probepackets:</dt> <dd>Probe packets are created by an IOAM encapsulating node by selecting some or all of the en route data packets and replicating them. A selected data packet that isreplicated,replicated and its (possibly truncated) copyisare forwarded with one or more IOAMoption,options, while the original packet isforwardedforwarded, normally, without IOAM options. To the extent possible, the original data packet and its replica are forwarded through the same path. The replica includes a Trace Option-Type that has its Active flag set, indicating that the IOAM decapsulating node should terminate it. In thiscasecase, the IOAM Active flag ensures that the replicated traffic is not forwarded beyond theIOAM domain.</t> </list></t>IOAM-Domain.</dd> </dl> </section> <section anchor="ioam-brown-field"title="Brownnumbered="true" toc="default"> <name>Brown Field Deployments:IOAM Unaware Nodes">IOAM-Unaware Nodes</name> <t>A network can consist of a mix ofIOAM awareIOAM-aware andIOAM unawareIOAM-unaware nodes. The encapsulation of IOAM-Data-Fields into different protocols (see also <xreftarget="ioam-encap"/>)target="ioam-encap" format="default"/>) are defined such that data packets that include IOAM-Data-Fields do not get dropped byIOAM unawareIOAM-unaware nodes. For example, packetswhichthat contain theIOAM-Trace-Option-TypesIOAM Trace Option-Types in IPv6Hop by HopHop-by-Hop extension headers are defined with bits to indicate "00 - skip over this option and continue processing the header". This will ensure that whenaan IOAM-unaware nodethat is IOAM unawarereceives a packet with IOAM-Data-Fields included, it does not drop the packet.</t> <t>Deploymentswhichthat leverage theIOAM-Trace-Option-Type(s)IOAM Trace Option-Type(s) could benefit from the ability to detect the presence ofIOAM unawareIOAM-unaware nodes,i.e.i.e., nodeswhichthat forward the packet but do notupdate/addupdate or add IOAM-Data-Fields inIOAM-Trace-Option-Type(s).IOAM Trace Option-Types. The node data that is defined as part of theIOAM-Trace-Option-Type(s)IOAM Trace Option-Type(s) includes a Hop_Lim field associated to the node identifier to detect missed nodes,i.e.i.e., "holes" in the trace. Monitoring/Analyticssystem(s)systems could utilize this information to account for the presence ofIOAM unawareIOAM-unaware nodes in the network.</t> </section> </section> <sectiontitle="IOAM Manageability">numbered="true" toc="default"> <name>IOAM Manageability</name> <t>The YANG model for configuring IOAM in network nodeswhichthat support IOAM is defined in <xreftarget="I-D.zhou-ippm-ioam-yang"/>.</t>target="I-D.ietf-ippm-ioam-yang" format="default"/>.</t> <t>A deployment can leverage IOAM profiles to limit the scope of IOAM features, allowing simpler implementation, verification, and interoperability testing in the context of specific use cases that do not require the full functionality of IOAM. An IOAM profile defines a use case or a set of use cases forIOAM,IOAM and an associated set of rules that restrict the scope and features of the IOAM specification, thereby limiting it to a subset of the full functionality. IOAM profiles are defined in <xreftarget="I-D.mizrahi-ippm-ioam-profile"/>.</t>target="I-D.mizrahi-ippm-ioam-profile" format="default"/>.</t> <t>For deployments where the IOAM capabilities of a node are unknown, <xreftarget="I-D.ietf-ippm-ioam-conf-state"/>target="RFC9359" format="default"/> could be used to discover the enabled IOAM capabilities of nodes. </t> </section> <section anchor="IANA"title="IANA Considerations">numbered="true" toc="default"> <name>IANA Considerations</name> <t>This documentdoes not request anyhas no IANA actions.</t> </section> <section anchor="Security"title="Security Considerations">numbered="true" toc="default"> <name>Security Considerations</name> <t>As discussed in <xreftarget="RFC7276"/>,target="RFC7276" format="default"/>, a successful attack on an OAM protocol ingeneral, and specificallygeneral and, specifically, onIOAM,IOAM can prevent the detection of failures oranomalies,anomalies or can create a false illusion of nonexistent ones.</t> <t>The Proof of Transit Option-Type (<xreftarget="IOAM-proof-of-transit"/>)target="IOAM-proof-of-transit" format="default"/>) is used for verifying the path of data packets. The security considerations of POT are further discussed in <xreftarget="I-D.ietf-sfc-proof-of-transit"/>.</t>target="I-D.ietf-sfc-proof-of-transit" format="default"/>.</t> <t>Security considerations related to the use of IOAM flags,in particularparticularly theloopback flagLoopback flag, are found in <xreftarget="RFC9322"/>.</t>target="RFC9322" format="default"/>.</t> <t>IOAM data can be subject to eavesdropping. Although the confidentiality of the user data is not at risk in this context, the IOAM data elements can be used for network reconnaissance, allowing attackers to collect information about network paths, performance, queue states, bufferoccupancyoccupancy, and other information. Recon is an improbable security threat in an IOAM deployment that is within a confined physical domain. However, in deployments that are not confined to a singleLAN,LAN but span multiple interconnected sites (for example, using an overlay network), the inter-site links are expected to be secured (e.g., by IPsec) in order to avoid external eavesdropping and introduction of malicious or false data. Another possible mitigation approach is to usethe "direct exporting" mode"Direct Exporting" <xreftarget="RFC9326"/>.target="RFC9326" format="default"/>. In thiscasecase, theIOAM relatedIOAM-related trace information would not be available in the customer datapackets,packets but would trigger the exporting of (secured) packet-related IOAM information at every node. IOAM data export and securing IOAM data export is outside the scope of this document.</t> <t>IOAM can be used as a means for implementingDenial of Service (DoS) attacks,orforamplifyingthem.Denial-of-Service (DoS) attacks. For example, a malicious attacker can add an IOAM header to packets or modify an IOAM header in en route packets in order to consume the resources of network devices that take part in IOAM or collectors that analyze the IOAM data. Another example is apacket lengthpacket-length attack, in which an attacker pushes headers associated with IOAM Option-Types into data packets, causing these packets to be increased beyond the MTU size, resulting in fragmentation or in packet drops. Such DoS attacks can be mitigated by deploying IOAM in confined administrativedomains,domains and by limiting the rate and/or the percentage of packets that an IOAM encapsulating node adds IOAM informationto,to as well as limiting rate and/or percentage of packets that an IOAM transit or an IOAM decapsulating node creates to export IOAM information extracted from the data packets that carry IOAM information.</t> <t>Even though IOAM focused on limited domains <xreftarget="RFC8799"/>,target="RFC8799" format="default"/>, there might be deployments for which it is important for IOAM transit nodes and IOAM decapsulating nodes to know that the data receivedhadn'thaven't been tampered with. In those cases, the IOAM data should be integrity protected. Integrity protection of IOAM data fields is described in <xreftarget="I-D.ietf-ippm-ioam-data-integrity"/>.target="I-D.ietf-ippm-ioam-data-integrity" format="default"/>. In addition, since IOAM options may include timestamps, if network devices use synchronizationprotocolsprotocols, then any attack on the time protocol <xreftarget="RFC7384"/>target="RFC7384" format="default"/> can compromise the integrity of the timestamp-related data fields. Synchronization attacks can be mitigated by combining a secured time distribution scheme, e.g., <xreftarget="RFC8915"/>,target="RFC8915" format="default"/>, and by using redundant clock sources <xreftarget="RFC5905"/>target="RFC5905" format="default"/> and/or redundant network paths for the time distribution protocol <xreftarget="RFC8039"/>.target="RFC8039" format="default"/>. </t> <t>At the management plane, attacks may be implemented by misconfiguring or by maliciously configuring IOAM-enabled nodes in a way that enables other attacks. Thus, IOAM configuration should be secured in a way that authenticates authorized users and verifies the integrity of configuration procedures.</t> <t>Notably, IOAM is expected to be deployed in limited network domains(<xref target="RFC8799"/>), thus<xref target="RFC8799" format="default"/>, thus, confining the potential attack vectorstowithin the limited domain. Indeed, in order to limit the scope of threatstowithin the current networkdomaindomain, the network operator is expected to enforce policies that prevent IOAM traffic from leaking outside theIOAM domain,IOAM-Domain and prevent an attacker from introducing malicious or false IOAM data to be processed and used within theIOAM domain.IOAM-Domain. IOAM data leakage could lead to privacy issues. Consider an IOAM encapsulating node that is a home gateway in an operator's network. A home gateway is often identified with anindividual, and revealingindividual. Revealing IOAMdatadata, such as "IOAM node identifier" or geolocation information outside of the limiteddomaindomain, could be harmful for that user. Note thattheDirectExport modeExporting <xreftarget="RFC9326"/>target="RFC9326" format="default"/> can mitigate the potential threat of IOAM data leaking through data packets.</t> </section><section title="Acknowledgements"> <t>The authors would like to thank Tal Mizrahi, Eric Vyncke, Nalini Elkins, Srihari Raghavan, Ranganathan T S, Barak Gafni, Karthik Babu Harichandra Babu, Akshaya Nadahalli, LJ Wobker, Erik Nordmark, Vengada Prasad Govindan, Andrew Yourtchenko, Aviv Kfir, Tianran Zhou, Zhenbin (Robin), Joe Clarke, Al Morton, Tom Herbet, Haoyu song, and Mickey Spiegel for the comments and advice on IOAM.</t> </section></middle><!-- *****BACK MATTER ***** --><back><!-- References split into informative and normative --> <!-- There are 2 ways to insert reference entries from the citation libraries: 1. define an ENTITY at the top, and use "ampersand character"RFC2629; here (as shown) 2. simply use a PI "less than character"?rfc include="reference.RFC.2119.xml"?> here (for I-Ds: include="reference.I-D.narten-iana-considerations-rfc2434bis.xml") Both are cited textually in the same manner: by using xref elements. If you use the PI option, xml2rfc will, by default, try to find included files in the same directory as the including file. You can also define the XML_LIBRARY environment variable with a value containing a set of directories to search. These can be either in the local filing system or remote ones accessed by http (http://domain/dir/... ).--> <!-- &RFC5905; --> <!--<displayreference target="I-D.ietf-nvo3-vxlan-gpe" to="VXLAN-GPE"/> <displayreference target="I-D.ietf-ippm-ioam-yang" to="IOAM-YANG"/> <displayreference target="I-D.mizrahi-ippm-ioam-profile" to="IOAM-PROFILES"/> <displayreference target="I-D.spiegel-ippm-ioam-rawexport" to="IOAM-RAWEXPORT"/> <displayreference target="I-D.ietf-sfc-proof-of-transit" to="PROOF-OF-TRANSIT"/> <displayreference target="I-D.ietf-ippm-ioam-ipv6-options" to="IOAM-IPV6-OPTIONS"/> <displayreference target="I-D.ietf-ippm-ioam-data-integrity" to="IOAM-DATA-INTEGRITY"/> <displayreference target="I-D.weis-ippm-ioam-eth" to="IOAM-ETH"/> <displayreference target="I-D.ietf-sfc-ioam-nsh" to="IOAM-NSH"/> <displayreference target="I-D.xzlnp-bier-ioam" to="BIER-IOAM"/> <displayreference target="I-D.brockners-ippm-ioam-geneve" to="IOAM-GENEVE"/> <displayreference target="I-D.gandhi-mpls-ioam" to="MPLS-IOAM"/> <displayreference target="I-D.ali-spring-ioam-srv6" to="IOAM-SRV6"/> <displayreference target="I-D.brockners-ippm-ioam-vxlan-gpe" to="IOAM-VXLAN-GPE"/> <references> <name>Informative References</name> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7665.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7799.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7384.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7276.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8300.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8915.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.5905.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8039.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8799.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9197.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9322.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9326.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8279.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8926.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.2784.xml"/> <referenceanchor="POSIX" target="https://standards.ieee.org/findstds/standard/1003.1-2008.html">anchor="I-D.ietf-nvo3-vxlan-gpe" target="https://datatracker.ietf.org/doc/html/draft-ietf-nvo3-vxlan-gpe-12"> <front><title>IEEE Std 1003.1-2008 (Revision of IEEE Std 1003.1-2004) - IEEE Standard<title>Generic Protocol Extension forInformation Technology - Portable Operating System Interface (POSIX(R))</title> <author> <organization>Institute of Electrical and Electronics Engineers</organization>VXLAN (VXLAN-GPE)</title> <author initials="F." surname="Maino" fullname="Fabio Maino" role="editor"> <organization>Cisco Systems</organization> </author> <author initials="L." surname="Kreeger" fullname="Larry Kreeger" role="editor"> <organization>Arrcus</organization> </author> <author initials="U." surname="Elzur" fullname="Uri Elzur" role="editor"> <organization>Intel</organization> </author> <dateyear="2008"/>month="September" day="22" year="2021"/> </front> <seriesInfoname="" value="IEEE Std 1003.1-2008"/>name="Internet-Draft" value="draft-ietf-nvo3-vxlan-gpe-12"/> </reference> <referenceanchor="IEEE1588v2" target="http://standards.ieee.org/findstds/standard/1588-2008.html">anchor="I-D.ietf-ippm-ioam-yang" target="https://datatracker.ietf.org/doc/html/draft-ietf-ippm-ioam-yang-06"> <front><title>IEEE Std 1588-2008 - IEEE Standard<title>A YANG Data Model fora Precision Clock SynchronizationIn-Situ OAM</title> <author initials="T." surname="Zhou" fullname="Tianran Zhou" role="editor"> <organization>Huawei</organization> </author> <author initials="J." surname="Guichard" fullname="Jim Guichard"> <organization>Futurewei</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners"> <organization>Cisco Systems</organization> </author> <author initials="S." surname="Raghavan" fullname="Srihari Raghavan"> <organization>Cisco Systems</organization> </author> <date month="February" day="27" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-ippm-ioam-yang-06"/> </reference> <reference anchor="I-D.mizrahi-ippm-ioam-profile" target="https://datatracker.ietf.org/doc/html/draft-mizrahi-ippm-ioam-profile-06"> <front> <title>In Situ OAM Profiles</title> <author initials="T." surname="Mizrahi" fullname="Tal Mizrahi"> <organization>Huawei</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners"> <organization>Cisco</organization> </author> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari" role="editor"> <organization>Thoughtspot</organization> </author> <author initials="R." surname="Sivakolundu" fullname="Ramesh Sivakolundu"> <organization>Cisco</organization> </author> <author initials="C." surname="Pignataro" fullname="Carlos Pignataro"> <organization>Cisco</organization> </author> <author initials="A." surname="Kfir" fullname="Aviv Kfir"> <organization>Nvidia</organization> </author> <author initials="B." surname="Gafni" fullname="Barak Gafni"> <organization>Nvidia</organization> </author> <author initials="M." surname="Spiegel" fullname="Mickey Spiegel"> <organization>Barefoot Networks</organization> </author> <author initials="T." surname="Zhou" fullname="Tianran Zhou"> <organization>Huawei</organization> </author> <date month="February" day="17" year="2022"/> </front> <seriesInfo name="Internet-Draft" value="draft-mizrahi-ippm-ioam-profile-06"/> </reference> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/reference.I-D.spiegel-ippm-ioam-rawexport.xml"/> <reference anchor="I-D.ietf-sfc-proof-of-transit" target="https://datatracker.ietf.org/doc/html/draft-ietf-sfc-proof-of-transit-08"> <front> <title>Proof of Transit</title> <author initials="F." surname="Brockners" fullname="Frank Brockners" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari" role="editor"> <organization>Thoughtspot</organization> </author> <author initials="T." surname="Mizrahi" fullname="Tal Mizrahi" role="editor"> <organization>Huawei Network.IO Innovation Lab</organization> </author> <author initials="S." surname="Dara" fullname="Sashank Dara"> <organization>Seconize</organization> </author> <author initials="S." surname="Youell" fullname="Stephen Youell"> <organization>JP Morgan Chase</organization> </author> <date month="October" day="31" year="2020"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-sfc-proof-of-transit-08"/> </reference> <reference anchor="I-D.ietf-ippm-ioam-ipv6-options" target="https://datatracker.ietf.org/doc/html/draft-ietf-ippm-ioam-ipv6-options-10"> <front> <title>In-situ OAM IPv6 Options</title> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari" role="editor"> <organization>Thoughtspot</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <date month="February" day="28" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-ippm-ioam-ipv6-options-10"/> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9359.xml"/> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/reference.I-D.ietf-ippm-ioam-data-integrity.xml"/> <reference anchor="I-D.weis-ippm-ioam-eth" target="https://datatracker.ietf.org/doc/html/draft-weis-ippm-ioam-eth-05"> <front> <title> EtherType Protocolfor Networked Measurement and Control Systems</title> <author> <organization>InstituteIdentification ofElectrical and Electronics Engineers</organization>In-situ OAM Data </title> <author initials="B." surname="Weis" fullname="Brian Weis" role="editor"> <organization>Independent</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="C." surname="Hill" fullname="Craig Hill"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari"> <organization>Thoughtspot</organization> </author> <author initials="V." surname="Govindan" fullname="Vengada Prasad Govindan"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="C." surname="Pignataro" fullname="Carlos Pignataro" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="N." surname="Nainar" fullname="Nagendra Kumar Nainar" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="H." surname="Gredler" fullname="Hannes Gredler"> <organization>RtBrick Inc.</organization> </author> <author initials="J." surname="Leddy" fullname="John Leddy"> </author> <author initials="S." surname="Youell" fullname="Stephen Youell"> <organization>JP Morgan Chase</organization> </author> <author initials="T." surname="Mizrahi" fullname="Tal Mizrahi"> <organization>Huawei Network.IO Innovation Lab</organization> </author> <author initials="A." surname="Kfir" fullname="Aviv Kfir"> <organization>Nvidia</organization> </author> <author initials="B." surname="Gafni" fullname="Barak Gafni"> <organization>Nvidia</organization> </author> <author initials="P." surname="Lapukhov" fullname="Petr Lapukhov"> <organization>Facebook</organization> </author> <author initials="M." surname="Spiegel" fullname="Mickey Spiegel"> <organization>Barefoot Networks, an Intel company</organization> </author> <dateyear="2008"/>month="February" day="21" year="2022"/> </front> <seriesInfoname="" value="IEEE Std 1588-2008"/>name="Internet-Draft" value="draft-weis-ippm-ioam-eth-05"/> </reference> <reference anchor="I-D.ietf-sfc-ioam-nsh" target="https://datatracker.ietf.org/doc/html/draft-ietf-sfc-ioam-nsh-11"> <front> <title> Network Service Header (NSH) Encapsulation for In-situ OAM (IOAM) Data </title> <author initials="F." surname="Brockners" fullname="Frank Brockners" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari" role="editor"> <organization>Thoughtspot</organization> </author> <date month="September" day="30" year="2022"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-sfc-ioam-nsh-11"/> </reference> <reference anchor="I-D.xzlnp-bier-ioam" target="https://datatracker.ietf.org/doc/html/draft-xzlnp-bier-ioam-05"> <front> <title>BIER Encapsulation for IOAM Data</title> <author initials="X." surname="Min" fullname="Xiao Min"> <organization>ZTE Corp.</organization> </author> <author initials="Z." surname="Zhang" fullname="Zheng Zhang"> <organization>ZTE Corp.</organization> </author> <author initials="Y." surname="Liu" fullname="Yisong Liu"> <organization>China Mobile</organization> </author> <author initials="N." surname="Nainar" fullname="Nagendra Nainar"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="C." surname="Pignataro" fullname="Carlos Pignataro"> <organization>Cisco Systems, Inc.</organization> </author> <date month="January" day="27" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-xzlnp-bier-ioam-05"/> </reference> <reference anchor="I-D.brockners-ippm-ioam-geneve" target="https://datatracker.ietf.org/doc/html/draft-brockners-ippm-ioam-geneve-05"> <front> <title>Geneve encapsulation for In-situ OAM Data</title> <author initials="F." surname="Brockners" fullname="Frank Brockners" role="editor"> <organization>Cisco</organization> </author> <author initials="S." surname="Bhandari" fullname="Shwetha Bhandari"> <organization>Thoughtspot</organization> </author> <author initials="V." surname="Govindan" fullname="Vengada Prasad Govindan"> <organization>Cisco</organization> </author> <author initials="C." surname="Pignataro" fullname="Carlos Pignataro" role="editor"> <organization>Cisco</organization> </author> <author initials="N." surname="Nainar" fullname="Nagendra Nainar" role="editor"> <organization>Cisco</organization> </author> <author initials="H." surname="Gredler" fullname="Hannes Gredler"> <organization>RtBrick Inc.</organization> </author> <author initials="J." surname="Leddy" fullname="John Leddy"> </author> <author initials="S." surname="Youell" fullname="Stephen Youell"> <organization>JMPC</organization> </author> <author initials="T." surname="Mizrahi" fullname="Tal Mizrahi"> <organization>Huawei Network.IO Innovation Lab</organization> </author> <author initials="P." surname="Lapukhov" fullname="Petr Lapukhov"> <organization>Facebook</organization> </author> <author initials="B." surname="Gafni" fullname="Barak Gafni"> <organization>Mellanox Technologies</organization> </author> <author initials="A." surname="Kfir" fullname="Aviv Kfir"> <organization>Mellanox Technologies</organization> </author> <author initials="M." surname="Spiegel" fullname="Mickey Spiegel"> <organization>Barefoot Networks</organization> </author> <date month="November" day="19" year="2020"/> </front> <seriesInfo name="Internet-Draft" value="draft-brockners-ippm-ioam-geneve-05"/> </reference> <reference anchor="I-D.gandhi-mpls-ioam" target="https://datatracker.ietf.org/doc/html/draft-gandhi-mpls-ioam-10"> <front> <title>MPLS Data Plane Encapsulation for In Situ OAM Data</title> <author initials="R." surname="Gandhi" fullname="Rakesh Gandhi" role="editor"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners"> <organization>Cisco Systems, Inc.</organization> </author> <author initials="B." surname="Wen" fullname="Bin Wen"> <organization>Comcast</organization> </author> <author initials="B." surname="Decraene" fullname="Bruno Decraene"> <organization>Orange</organization> </author> <author initials="H." surname="Song" fullname="Haoyu Song"> <organization>Futurewei Technologies</organization> </author> <date month="March" day="10" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-gandhi-mpls-ioam-10"/> </reference> <reference anchor="I-D.ali-spring-ioam-srv6" target="https://datatracker.ietf.org/doc/html/draft-ali-spring-ioam-srv6-06"> <front> <title> Segment Routing Header encapsulation for In-situ OAM Data </title> <author initials="Z." surname="Ali" fullname="Zafar Ali"> <organization>Cisco Systems</organization> </author> <author initials="R." surname="Gandhi" fullname="Rakesh Gandhi"> <organization>Cisco Systems</organization> </author> <author initials="C." surname="Filsfils" fullname="Clarence Filsfils"> <organization>Cisco Systems</organization> </author> <author initials="F." surname="Brockners" fullname="Frank Brockners"> <organization>Cisco Systems</organization> </author> <author initials="N." surname="Nainar" fullname="Nagendra Nainar"> <organization>Cisco Systems</organization> </author> <author initials="C." surname="Pignataro" fullname="Carlos Pignataro"> <organization>Cisco Systems</organization> </author> <author initials="C." surname="Li" fullname="Cheng Li"> <organization>Huawei</organization> </author> <author initials="M." surname="Chen" fullname="Mach Chen"> <organization>Huawei</organization> </author> <author initials="G." surname="Dawra" fullname="Gaurav Dawra"> <organization>LinkedIn</organization> </author> <date month="July" day="10" year="2022"/> </front> <seriesInfo name="Internet-Draft" value="draft-ali-spring-ioam-srv6-06"/> </reference> <reference anchor="I-D.brockners-ippm-ioam-vxlan-gpe" target="https://datatracker.ietf.org/doc/html/draft-brockners-ippm-ioam-vxlan-gpe-03"> <front> <title>VXLAN-GPE Encapsulation for In-situ OAM Data </title> <author initials="F." surname="Brockners" fullname="F. Brockners"> <organization></organization> </author> <author initials="S." surname="Bhandari" fullname="S. Bhandari"> <organization></organization> </author> <author initials="V." surname="Govindan" fullname="V. Govindan"> <organization></organization> </author> <author initials="C." surname="Pignataro" fullname="C. Pignataro"> <organization>Cisco</organization> </author> <author initials="H." surname="Gredler" fullname="H. Gredler"> <organization>RtBrick Inc.</organization> </author> <author initials="J." surname="Leddy" fullname="J. Leddy"> <organization></organization> </author> <author initials="S." surname="Youell" fullname="S. Youell"> <organization>JMPC</organization> </author> <author initials="T." surname="Mizrahi" fullname="T. Mizrahi"> <organization>Huawei Network.IO Innovation Lab</organization> </author> <author initials="A." surname="Kfir" fullname="A. Kfir"> <organization></organization> </author> <author initials="B." surname="Gafni" fullname="B. Gafni"> <organization>Mellanox Technologies, Inc.</organization> </author> <author initials="P." surname="Lapukhov" fullname="P. Lapukhov"> <organization>Facebook</organization> </author> <author initials="M." surname="Spiegel" fullname="M. Spiegel"> <organization></organization> </author> <date month="November" day="4" year="2019"/> </front> <seriesInfo name="Internet-Draft" value="draft-brockners-ipxpm-ioam-vxlan-gpe-03"/> </reference>--> <references title="Informative References"> &RFC7665; &RFC7799; &RFC7384; &RFC7276; &RFC8300; &RFC8915; &RFC5905; &RFC8039; &RFC8799; &RFC9197; &RFC9322; &RFC9326; &RFC8279; &RFC8926; &RFC2784; &I-D.ietf-nvo3-vxlan-gpe; &I-D.zhou-ippm-ioam-yang; &I-D.mizrahi-ippm-ioam-profile; &I-D.spiegel-ippm-ioam-rawexport; &I-D.ietf-sfc-proof-of-transit; &I-D.ietf-ippm-ioam-ipv6-options; &I-D.ietf-ippm-ioam-conf-state; &I-D.ietf-ippm-ioam-data-integrity; &I-D.weis-ippm-ioam-eth; &I-D.ietf-sfc-ioam-nsh; &I-D.xzlnp-bier-ioam; &I-D.brockners-ippm-ioam-geneve; &I-D.gandhi-spring-ioam-sr-mpls; &I-D.ali-spring-ioam-srv6; &I-D.brockners-ippm-ioam-vxlan-gpe;</references> <section numbered="false" toc="default"> <name>Acknowledgements</name> <t>The authors would like to thank <contact fullname="Tal Mizrahi"/>, <contact fullname="Eric Vyncke"/>, <contact fullname="Nalini Elkins"/>, <contact fullname="Srihari Raghavan"/>, <contact fullname="Ranganathan T S"/>, <contact fullname="Barak Gafni"/>, <contact fullname="Karthik Babu Harichandra Babu"/>, <contact fullname="Akshaya Nadahalli"/>, <contact fullname="LJ Wobker"/>, <contact fullname="Erik Nordmark"/>, <contact fullname="Vengada Prasad Govindan"/>, <contact fullname="Andrew Yourtchenko"/>, <contact fullname="Aviv Kfir"/>, <contact fullname="Tianran Zhou"/>, <contact fullname="Zhenbin (Robin)"/>, <contact fullname="Joe Clarke"/>, <contact fullname="Al Morton"/>, <contact fullname="Tom Herbet"/>, <contact fullname="Haoyu Song"/>, and <contact fullname="Mickey Spiegel"/> for the comments and advice on IOAM.</t> </section> </back> </rfc>