libswscale/ops.h - master - githubmirror/FFmpeg

libswscale/ops.h
375 строк13 KB
Niklas Haas
swscale/ops: add SWS_OP_LUT_3D and supporting code
03 авг 2026, 12:32
03 авг 2026, 12:326c770f3
/**
 * Copyright (C) 2025 Niklas Haas
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#ifndef SWSCALE_OPS_H
#define SWSCALE_OPS_H

#include <assert.h>
#include <stdbool.h>
#include <stdalign.h>

#include "libavutil/bprint.h"

#include "graph.h"
#include "filters.h"
#include "lut3d.h"
#include "rational64.h"
#include "uops.h"

typedef enum SwsOpType {
    SWS_OP_INVALID = 0,

    /* Defined for all types; but implemented for integers only */
    SWS_OP_READ,            /* gather raw pixels from planes */
    SWS_OP_WRITE,           /* write raw pixels to planes */
    SWS_OP_SWAP_BYTES,      /* swap byte order (for differing endianness) */
    SWS_OP_SWIZZLE,         /* rearrange channel order, or duplicate channels */

    /* Bit manipulation operations. Defined for integers only. */
    SWS_OP_UNPACK,          /* split tightly packed data into components */
    SWS_OP_PACK,            /* compress components into tightly packed data */
    SWS_OP_LSHIFT,          /* logical left shift of raw pixel values */
    SWS_OP_RSHIFT,          /* right shift of raw pixel values */

    /* Generic arithmetic. Defined and implemented for all types */
    SWS_OP_CLEAR,           /* clear pixel values */
    SWS_OP_CONVERT,         /* convert (cast) between formats */
    SWS_OP_MIN,             /* numeric minimum */
    SWS_OP_MAX,             /* numeric maximum */
    SWS_OP_SCALE,           /* multiplication by scalar */

    /* Floating-point only arithmetic operations. */
    SWS_OP_LINEAR,          /* generalized linear affine transform */
    SWS_OP_DITHER,          /* add dithering noise */

    /* Filtering operations. */
    SWS_OP_FILTER_H,        /* horizontal filtering */
    SWS_OP_FILTER_V,        /* vertical filtering */

    /* Table-based operations. Defined for floating point types only. */
    SWS_OP_LUT_3D,          /* apply a SwsLut3D */

    SWS_OP_TYPE_NB,
} SwsOpType;

const char *ff_sws_op_type_name(SwsOpType op);

/* Compute SwsCompMask from values with denominator != 0 */
SwsCompMask ff_sws_comp_mask_q4(const AVRational64 q[4]);

typedef enum SwsCompFlags {
    SWS_COMP_GARBAGE = 1 << 0, /* contents are undefined / garbage data */
    SWS_COMP_EXACT   = 1 << 1, /* value is an exact integer */
    SWS_COMP_ZERO    = 1 << 2, /* known to be a constant zero */
    SWS_COMP_SWAPPED = 1 << 3, /* byte order is swapped */
    SWS_COMP_COPY    = 1 << 4, /* value is unmodified from the source plane */
    SWS_COMP_CONST   = 1 << 5, /* value is a fixed constant */
} SwsCompFlags;

typedef struct SwsComps {
    SwsCompFlags flags[4]; /* knowledge about (output) component contents */

    /* Keeps track of the known possible value range, or {0, 0} for undefined
     * or (unknown range) floating point inputs */
    AVRational64 min[4], max[4];

    /* Keeps track of input (forward) and output (reverse) dependencies */
    SwsCompMask dep_in[4], dep_out[4];
} SwsComps;

typedef enum SwsReadWriteMode {
    /**
     * Note: 1-component reads are either SWS_RW_PLANAR or SWS_RW_PACKED,
     * depending on the underlying interpretation. If multiple components are
     * packed into one element (e.g. rgb10a2 -> u16), they are marked as
     * SWS_RW_PACKED. Otherwise (e.g. gray16le), they are SWS_RW_PLANAR.
     *
     * This is a purely semantic/informative difference; the underlying code
     * treats 1-components reads/writes the same regardless of mode.
     */
    SWS_RW_PLANAR,  /* one plane per component */
    SWS_RW_PACKED,  /* all components on a single plane */
    SWS_RW_PALETTE, /* plane 0 is 8-bit index, plane 1 is packed palette */
} SwsReadWriteMode;

typedef struct SwsReadWriteOp {
    /**
     * Examples:
     *   rgba      = 4x u8 packed
     *   yuv444p   = 3x u8
     *   rgb565    = 1x u16   <- use SWS_OP_UNPACK to unpack
     *   monow     = 1x u8 (frac 3)
     *   rgb4      = 1x u8 (frac 1)
     *   pal8      = 4x u8 (palette)
     */
    SwsReadWriteMode mode; /* how data is laid out in memory */
    uint8_t elems; /* number of elements (of type `op.type`) to read/write */
    uint8_t frac;  /* fractional pixel step factor (log2) */

    /**
     * Filter kernel to apply to each plane while sampling. Currently, only
     * one shared filter kernel is supported for all planes. (Optional)
     *
     * Note: As with SWS_OP_FILTER_*, if a filter kernel is in use, the read
     * operation will always output floating point values.
     */
    struct {
        SwsOpType op;               /* some value of SWS_OP_FILTER_* */
        SwsFilterWeights *kernel;   /* (refstruct) */
        SwsPixelType type;          /* pixel type to store result as */
    } filter;
} SwsReadWriteOp;

typedef struct SwsPackOp {
    /**
     * Packed bits are assumed to be LSB-aligned within the underlying
     * integer type; i.e. (msb) 0 ... X Y Z W (lsb).
     */
    uint8_t pattern[4]; /* bit depth pattern, from MSB to LSB */
} SwsPackOp;

typedef struct SwsSwizzleOp {
    /**
     * Input component for each output component:
     *   Out[x] := In[swizzle.in[x]]
     */
    union {
        uint32_t mask;
        uint8_t in[4];
        struct { uint8_t x, y, z, w; };
    };
} SwsSwizzleOp;

#define SWS_SWIZZLE(X,Y,Z,W) ((SwsSwizzleOp) { .in = {X, Y, Z, W} })
void ff_sws_comp_mask_swizzle(SwsCompMask *mask, const SwsSwizzleOp *swiz);

typedef struct SwsShiftOp {
    uint8_t amount; /* number of bits to shift */
} SwsShiftOp;

typedef struct SwsClearOp {
    SwsCompMask mask;    /* mask of components to clear */
    AVRational64 value[4]; /* value to set */
} SwsClearOp;

typedef struct SwsConvertOp {
    SwsPixelType to; /* type of pixel to convert to */
    bool expand; /* if true, integers are expanded to the full range */
} SwsConvertOp;

typedef struct SwsClampOp {
    AVRational64 limit[4]; /* per-component min/max value */
} SwsClampOp;

typedef struct SwsScaleOp {
    AVRational64 factor; /* scalar multiplication factor */
} SwsScaleOp;

typedef struct SwsDitherOp {
    AVRational64 *matrix; /* tightly packed dither matrix (refstruct) */
    AVRational64 min, max; /* minimum/maximum value in `matrix` */
    int size_log2; /* size (in bits) of the dither matrix */
    int8_t y_offset[4]; /* row offset for each component, or -1 for ignored */
} SwsDitherOp;

typedef struct SwsLinearOp {
    /**
     * Generalized 5x5 affine transformation:
     *   [ Out.x ] = [ A B C D E ]
     *   [ Out.y ] = [ F G H I J ] * [ x y z w 1 ]
     *   [ Out.z ] = [ K L M N O ]
     *   [ Out.w ] = [ P Q R S T ]
     */
    AVRational64 m[4][5];
} SwsLinearOp;

/* m[i][j] <-> 1 << (5 * i + j) */
uint32_t ff_sws_linear_mask(const SwsLinearOp *c);

typedef struct SwsFilterOp {
    SwsFilterWeights *kernel; /* filter kernel (refstruct) */
    SwsPixelType type;        /* pixel type to store result as */
} SwsFilterOp;

typedef struct SwsLut3dOp {
    /**
     * Reference to the external LUT3D to apply. This is managed by the caller,
     * and must remain valid for the lifetime of the SwsOp and any compiled
     * functions derived from it.
     *
     * *lut is never dereferenced by the SwsOp code itself, only at runtime by
     * the actual dispatched implementation, and may be freely modified even
     * after op compilation to place new values for dynamic tone-mapping.
     *
     * The reference algorithm for this operation lives in lut3d.c, and
     * includes a tetrahedral interpolation component for the input LUT, and
     * then an optional linear tone mapping LUT plus trilinear output LUT
     * (when lut->dynamic is true).
     *
     * All linear interpolations are performed in the pixel value's native
     * representation, even though the LUTs themselves are stored as unsigned
     * packed 16-bit integers. The input value range is assumed to be scaled
     * and clamped to the LUT's domain (i.e. [0, INPUT_LUT_SIZE - 1]), and the
     * output value range will be [0, 2^16-1], except for the alpha channel,
     * which is passed through untouched.
     */
    const SwsLut3D *lut; /* refstruct */
    bool dynamic;
} SwsLut3dOp;

typedef struct SwsOp {
    SwsOpType op;      /* operation to perform */
    SwsPixelType type; /* pixel type to operate on */
    union {
        SwsLinearOp     lin;
        SwsReadWriteOp  rw;
        SwsPackOp       pack;
        SwsSwizzleOp    swizzle;
        SwsShiftOp      shift;
        SwsClearOp      clear;
        SwsConvertOp    convert;
        SwsClampOp      clamp;
        SwsScaleOp      scale;
        SwsDitherOp     dither;
        SwsFilterOp     filter;
        SwsLut3dOp      lut3d;
    };

    /**
     * Metadata about the operation's input/output components. Discarded
     * and regenerated automatically by `ff_sws_op_list_update_comps()`.
     *
     * Note that backends may rely on the presence and accuracy of this
     * metadata for all operations, during ff_sws_ops_compile().
     */
    SwsComps comps;
} SwsOp;

#define SWS_OP_NEEDED(op, idx) (!((op)->comps.flags[idx] & SWS_COMP_GARBAGE))

/* Compute SwsCompMask from a mask of needed components */
SwsCompMask ff_sws_comp_mask_needed(const SwsOp *op);

/**
 * Return the number of planes involved in a read/write operation.
 */
int ff_sws_rw_op_planes(const SwsOp *op);

/**
 * Describe an operation in human-readable form.
 */
void ff_sws_op_desc(AVBPrint *bp, const SwsOp *op);

/**
 * Frees any allocations associated with an SwsOp and sets it to {0}.
 */
void ff_sws_op_uninit(SwsOp *op);

/**
 * Apply an operation to an AVRational64. No-op for read/write operations.
 */
void ff_sws_apply_op_q(const SwsOp *op, AVRational64 x[4]);

/**
 * Helper struct for representing a list of operations.
 */
typedef struct SwsOpList {
    SwsOp *ops;
    int num_ops;

    /* Metadata associated with this operation list */
    SwsFormat src, dst;

    /* Input/output plane indices */
    uint8_t plane_src[4], plane_dst[4];

    /**
     * Source component metadata associated with pixel values from each
     * corresponding component (in plane/memory order, i.e. not affected by
     * `plane_src`). Lets the optimizer know additional information about
     * the value range and/or pixel data to expect.
     *
     * The default value of {0} is safe to pass in the case that no additional
     * information is known.
     */
    SwsComps comps_src;
} SwsOpList;

SwsOpList *ff_sws_op_list_alloc(void);
void ff_sws_op_list_free(SwsOpList **ops);

/**
 * Returns a duplicate of `ops`, or NULL on OOM.
 */
SwsOpList *ff_sws_op_list_duplicate(const SwsOpList *ops);

/**
 * Returns the input operation for a given op list, or NULL if there is none
 * (e.g. for a pure CLEAR-only operation list).
 *
 * This will always be an op of type SWS_OP_READ.
 */
const SwsOp *ff_sws_op_list_input(const SwsOpList *ops);

/**
 * Returns the output operation for a given op list, or NULL if there is none.
 *
 * This will always be an op of type SWS_OP_WRITE.
 */
const SwsOp *ff_sws_op_list_output(const SwsOpList *ops);

/**
 * Returns whether an op list represents a true no-op operation, i.e. may be
 * eliminated entirely from an execution graph.
 */
bool ff_sws_op_list_is_noop(const SwsOpList *ops);

/**
 * Returns the size of the largest pixel type used in `ops`.
 */
int ff_sws_op_list_max_size(const SwsOpList *ops);

/**
 * These will take over ownership of `op` and set it to {0}, even on failure.
 */
int ff_sws_op_list_append(SwsOpList *ops, SwsOp *op);
int ff_sws_op_list_insert_at(SwsOpList *ops, int index, SwsOp *op);

void ff_sws_op_list_remove_at(SwsOpList *ops, int index, int count);

/**
 * Print out the contents of an operation list.
 */
void ff_sws_op_list_print(void *log_ctx, int log_level, int log_level_extra,
                          const SwsOpList *ops);

/**
 * Infer + propagate known information about components. Called automatically
 * when needed by the optimizer and compiler.
 */
void ff_sws_op_list_update_comps(SwsOpList *ops);

/**
 * Fuse compatible and eliminate redundant operations, as well as replacing
 * some operations with more efficient alternatives.
 */
int ff_sws_op_list_optimize(SwsOpList *ops);

#endif