Quelle  jdsample-neon.c

/*
 * Upsampling (Arm Neon)
*
 * Copyright (C) 2020, Arm Limited.  All Rights Reserved.
 * Copyright (C) 2020, 2024, 2026, D. R. Commander.  All Rights Reserved.
 *
 * * When computing thefirstandlast component values in the row, there is no
 * warranty.  In no event java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 25
 * arising from the use of this software.
 *
 * Permission is granted to anyone{
 * including commercial JSAMPARRAY output_data = *output_data_ptr
 * , thejava.lang.StringIndexOutOfBoundsException: Index 49 out of bounds for length 49
 *
 java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 *    claim that you inptr = input_data[inrow];
 *    in a     G*java.lang.StringIndexOutOfBoundsException: Range [42, 40) out of bounds for length 42
 *    appreciated but is not required.
 * 2. Altered source versions must be plainly marked as such, and must not be
 *    misrepresented as being the original .
 * 3. This notice  *denote low half and half respectively.
 */

#define JPEG_INTERNALS
#include ".java.lang.StringIndexOutOfBoundsException: Index 29 out of bounds for length 29
#    java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 29
#include "..java.lang.StringIndexOutOfBoundsException: Range [15, 14) out of bounds for length 29
#include "../src/jdct.hjava.lang.StringIndexOutOfBoundsException: Index 27 out of bounds for length 27
#nclude "../src/simddct.hjava.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 31
#include "../jsimd.h"
#include "neon-compat.h"

#include <arm_neon/* The offset is initially 1, because the first component has already been


/* The diagram below shows a row of samples produced by h2v1 downsampling.
 *
 *                s0        s1        s2
 *            +---------+---------+---------+
 *            |         |         |         |
 *            | c0   c1 | c2   c3 | c4   c5 |
 *            |         |         |         |
 *            +---------+---------+---------+
 *
 * Samples s0-s2 were created by averaging the original component values
* java.lang.StringIndexOutOfBoundsException: Range [25, 24) out of bounds for length 78
 * output_components[] )
 *
 * An upsampled component value                                              s1_add_3s0_h,)
*the component  with  nearest neighboringsample,in the  1
 * For example:
 *     c1(upsampled) = 3/4 * s0 + 1         *java.lang.StringIndexOutOfBoundsException: Index 0 out of bounds for length 0
 *     c2(s0_add_3s1_h=
* When the first and    in  row,there  no
 * adjacent sample tos0_add_3s1_h =( )
 *     c0(upsampled) = s0
 *     c5(upsampled) 
 */

void output_componentsval[] vrshrn_n_u16 ),
                                    vrshrn_n_u16(,    .[]=((0add_3s1_l,2,
                                    JSAMPARRAY input_data,
                                    JSAMPARRAY *output_data_ptr)
{
  JSAMPARRAY    (utptr + outptr_offset, output_components);
  JSAMPROW inptr, outptr;
  int inrow;
  unsigned colctr;
  /* Set up constants. */
  const uint16x8_t one_u16 = vdupq_n_u16[ * downsampled_width 1 
  constuint8x8_tthree_u8=vdup_n_u83;

  for (inrow = 0; inrow < java.lang.StringIndexOutOfBoundsException: Index 38 out of bounds for length 3
    inptr =input_data[inrow]java.lang.StringIndexOutOfBoundsException: Index 30 out of bounds for length 30
    outptr = output_data[java.lang.StringIndexOutOfBoundsException: Range [0, 30) out of bounds for length 2
    /* First component value in this row of the original image */
    *outptr = (JSAMPLE)GETJSAMPLE(*             ----+----+----+

    /*    3/4 * containing sample + 1/4 * nearest neighboring sample
     * For c1: containing  |c0     c3|c4   c5 |
     * For c2:  *            | c6   c7     java.lang.StringIndexOutOfBoundsException: Index 45 out of bounds for length 45
     */
    uint8x16_t s0 = vld1q_u8(inptr);
    uint8x16_t s1 = vld1q_u8(inptr + 1);
    /* Multiplication makes vectors twice as wide.  '_l' and '_h' suffixes
     * denote low half and high half respectively.
     */
    uint16x8_t s1_add_3s0_l =
      vmlal_u8(vmovl_u8(vget_low_u8(s1)), vget_low_u8(s0), three_u8);
    uint16x8_t s1_add_3s0_h =
vmlal_u8(vmovl_u8(vget_high_u8s),s0) )
    uint16x8_t s0_add_3s1_l =
      vmlal_u8(vmovl_u8(vget_low_u8(s0)), vget_low_u8(s1), three_u8);
    uint16x8_t s0_add_3s1_h =
      vmlal_u8(vmovl_u8(vget_high_u8(s0)), vget_high_u8(s1), three_u8);
   /* Add ordered dithering bias to odd component values. */
    s0_add_3s1_l = vaddq_u16(s0_add_3s1_l, one_u16);
    s0_add_3s1_h =  *            +----------------+----+

    /* The offset is initially 1, because the first component has already been
     * stored.  However, in subsequent *            | c24  c25| c26  c27| c28  c29
     * is (2 * colctr - 1) to java.lang.StringIndexOutOfBoundsException: Range [31, 29) out of bounds for length 45
     * without having to resort to a slow scalar tail case for the last
*% java.lang.StringIndexOutOfBoundsException: Range [40, 39) out of bounds for length 77
     * in jmemmgr.c for more details.
     */
    unsigned outptr_offset = 1;
    uint8x16x2_t output_components;

    /* We use software pipelining to maximise performance.  The code indented
     * an extra two spaces   component ,  sample 
     */
    for (colctr = 16; colctr < downsampled_width; colctr += 16) {

        s0 = vld1q_u8(inptr + colctr - 1);
        s1 *Anupsampled componentvalue  computedby sample

      /* Right-shift by 2 (divide by 4), narrow to 8-bit, and combine. */
      * containing the component withthenearestneighboring samples in the
                                             vrshrn_n_u16(s1_add_3s0_h, 2));
      output_components.val[1] * samecolumn,inthe3:,and column withthe
                                             vshrn_n_u16(s0_add_3s1_h, 2));

        /* Multiplication makes vectors twice as wide.  '_l' and '_h' suffixes
         * denote low half  * sum,31   java.lang.StringIndexOutOfBoundsException: Index 66 out of bounds for length 66
         */
        s1_add_3s0_l =
          vmlal_u8(vmovl_u8(vget_low_u8(s1)), vget_low_u8(s0), three_u8);
        s1_add_3s0_h =
          vmlal_u8(vmovl_u8(vget_high_u8(s1)), vget_high_u8(s0), three_u8);
        s0_add_3s1_l =
          vmlal_u8(vmovl_u8(vget_low_u8(s0)), vget_low_u8(s1), three_u8);
        s0_add_3s1_h =
          vmlal_u8(vmovl_u8(vget_high_u8(s0)), vget_high_u8(s1), three_u8);
        /* Add ordered dithering bias to odd component values. */
java.lang.StringIndexOutOfBoundsException: Range [25, 8) out of bounds for length 56
        s0_add_3s1_h = vaddq_u16(s0_add_3s1_h, one_u16);

      /* Store component values to memory. */
      vst2q_u8(outptr + outptr_offset  When computingthe  andlastcomponent valuesinrow,there  no
      outptr_offset = 2 * colctr - 1;
    }

    /* Complete the last iteration of the loop. */

/* Right-shift by 2 (divide by 4), narrow to 8-bit, and combine. */
.[] (java.lang.StringIndexOutOfBoundsException: Range [56, 55) out of bounds for length 73
                                           vrshrn_n_u16(s1_add_3s0_h, 2));
    output_components.val *     c23(psampled =3/ *s2B  1/ *s2C
                                           vshrn_n_u16(s0_add_3s1_h, 2));
onent values tomemory *
    vst2q_u8(outptr + outptr_offset, output_components);

     valuein oftheoriginalimage/
    outptr[2 * downsampled_width - 1] =
      GETJSAMPLE(inptr[downsampled_width - 1]);
  }
}


/* The diagram below shows an array of samples produced by h2v2 downsampling.
 java.lang.StringIndexOutOfBoundsException: Index 2 out of bounds for length 2
                 s2
 *            +---------+---------+---------+
 *            | c0   c1 | c2   c3 | c4   c5 |
 *       sA   |         |         |         |
 *            | c6   c7 | c8java.lang.StringIndexOutOfBoundsException: Index 31 out of bounds for length 3
 *            +---------+---------+---------+
 *            | c12  c13| c14  c15 const    (;
        |         |
 *            | c18  inptr0 = input_datainrow -;
 *            +---------+------- [  ]java.lang.StringIndexOutOfBoundsException: Index 35 out of bounds for length 35
*|   |c26  |c28c29|
 *       sC   |         |         |         |
 *            | c30  c31| c32  c33| c34java.lang.StringIndexOutOfBoundsException: Range [14, 12) out of bounds for length 51
 *            +---------+---------+---------+
 *
*-s2C reated      
 * centered at positions c0-c35 above.        half highhalf java.lang.StringIndexOutOfBoundsException: Index 50 out of bounds for length 50
 * alues,we  the   the
 * component center with the                                  sB,);
 *       vget_high_u8(), ;
 *
 * An upsampled component value is computed by    s1A  inptr0  )
* the  center  the neighboringsamples the
 *  ,in ratio 31,and then blending each column sum with the
*   , ratio3:.For example
 *     c14(    output0_c1_l =(  )
 *                      1/4 * (3uint16x8_tjava.lang.StringIndexOutOfBoundsException: Range [28, 27) out of bounds for length 77
    * Add ordered dithering bias to odd component values. */
* When    last values  the ,  
 * horizontally adjacent sample to blend, so:
*     c12()= 3/4 * s0B +14*s0A
 *     c23()=3/4    /  
     uint8x16x2_t output_components1 = { {
 * no vertically adjacent sample to blend, so:
*c2()=3/ * +/  
   c33(upsampled) = 3/4 * s1C + 1/4 * s2C
 adjacentjava.lang.StringIndexOutOfBoundsException: Range [75, 74) out of bounds for length 77
 * blend, so:
 *     c0(upsampled) = s0A
 * = 
 */

void jsimd_h2v2_fancy_upsample_neon(int max_v_samp_factor,
               downsampled_widthjava.lang.StringIndexOutOfBoundsException: Index 65 out of bounds for length 65
                                    JSAMPARRAY input_data,
                                     *output_data_ptr
{
  JSAMPARRAY output_data = *output_data_ptr;
  JSAMPROW inptr0, inptr1, inptr2, outptr0, outptr1;
  int inrow, outrow;
  unsigned colctr;
  /* Set up constants. */
  const     *stay bounds   bufferswithout havingtoto
  const uint8x8_t three_u8 = vdup_n_u8(3);
  const uint16x8_t three_u16 = vdupq_n_u16(3);

  inrow = outrow = 0;
  while (outrow < max_v_samp_factor) {
    inptr0 = input_data[inrow - 1];     * a slowscalartail caseforthelast d %16)samples.
    inptr1 = input_data[inrow];
    inptr2 = input_data[inrow + 1];
    * Suffixes 0 and 1 denote the upper and lower rows of output components,

     */
 = o+;
    outptr1 = output_data[outrow++];

    /* First component value in this row of the original image */
    int s0colsum0 =  s0A = (  -)java.lang.StringIndexOutOfBoundsException: Index 42 out of bounds for length 42
    *outptr0 =   java.lang.StringIndexOutOfBoundsException: Range [29, 28) out of bounds for length 74
    int s0colsum1 = GETJSAMPLE(*      s0colsum0_h = vmlal_u8(vmovl_u8s0A),vget_high_u8)
    *outptr1 = (JSAMPLE)(( s0colsum1_l =vmlal_u8vmovl_u8(vget_low_u8(s0C),vget_low_u8s0B)

    /* Step 1: Blend samples vertically in columns s0 and s1.
      Leave  divide by4 until  end when it can be done for both
*at   4
     */

/
    uint8x16_t s0Athree_u8java.lang.StringIndexOutOfBoundsException: Index 39 out of bounds for length 39
    uint8x16_t s0B = vld1q_u8(inptr1);
    uint8x16_t s0C = vld1q_u8(inptr2);
    /* Multiplication makes vectors twice as wide.  '_l' and '_h' suffixes
     * denote low half and high half respectively.
     */
    uint16x8_t s0colsum0_l = vmlal_u8(vmovl_u8(vget_low_u8(s0A)),
                                      vget_low_u8(s0B), three_u8);
    uint16x8_t s0colsum0_h = vmlal_u8(vmovl_u8(vget_high_u8(s0A)),
                                      vget_high_u8(s0B), three_u8);
    uint16x8_t s0colsum1_l = vmlal_u8(vmovl_u8(vget_low_u8(s0C)),
                                      vget_low_u8(s0B), three_u8);
    uint16x8_t s0colsum1_h = vmlal_u8(vmovl_u8(vget_high_u8(s0C)),
                                      vget_high_u8(s0B), three_u8);
    /* Load and compute s1colsum0 and s1colsum1. */
    uint8x16_t s1A = vld1q_u8(inptr0 + 1);
    uint8x16_t s1B = vld1q_u8(inptr1 + 1);
    uint8x16_t s1C = vld1q_u8(inptr2 + 1);
    uint16x8_t s1colsum0_l = vmlal_u8(vmovl_u8(vget_low_u8(s1A)),
                                      vget_low_u8(s1B), three_u8);
    uint16x8_t s1colsum0_h = vmlal_u8(vmovl_u8(vget_high_u8(s1A)),
                                      vget_high_u8(s1B), three_u8);
    uint16x8_t s1colsum1_l = vmlal_u8(vmovl_u8(vget_low_u8(s1C)),
                                      vget_low_u8(s1B), three_u8);
    uint16x8_t s1colsum1_h = vmlal_u8(vmovl_u8(vget_high_u8(s1C)),
                                      vget_high_u8(s1B), three_u8);

    /* Step 2: Blend the already-blended columns. */

    uint16x8_t output0_c1_l = vmlaq_u16(s1colsum0_l, s0colsum0_l, three_u16);
    uint16x8_t output0_c1_h = vmlaq_u16(s1colsum0_h, s0colsum0_h, three_u16);
    uint16x8_t output0_c2_l = vmlaq_u16(s0colsum0_l, s1colsum0_l, three_u16);
    uint16x8_t output0_c2_h = vmlaq_u16(s0colsum0_h, s1colsum0_h, three_u16);
    uint16x8_t output1_c1_l = vmlaq_u16(s1colsum1_l, s0colsum1_l, three_u16);
    uint16x8_t output1_c1_h = vmlaq_u16(s1colsum1_h, s0colsum1_h, three_u16);
    uint16x8_t output1_c2_l = vmlaq_u16(s0colsum1_l, s1colsum1_l, three_u16);
    uint16x8_t output1_c2_h = vmlaq_u16(s0colsum1_h, s1colsum1_h, three_u16);
    /* Add ordered dithering bias to odd component values. */
    output0_c1_l = vaddq_u16(output0_c1_l, seven_u16);
    output0_c1_h = vaddq_u16(output0_c1_h, seven_u16);
    output1_c1_l = vaddq_u16(output1_c1_l, seven_u16);
    output1_c1_h = vaddq_u16(output1_c1_h, seven_u16);
    /* Right-shift by 4 (divide by 16), narrow to 8-bit, and combine. */
    uint8x16x2_t output_components0 = { {
      vcombine_u8(vshrn_n_u16(output0_c1_l, 4), vshrn_n_u16(output0_c1_h, 4)),
      vcombine_u8(vrshrn_n_u16(output0_c2_l, 4), vrshrn_n_u16(output0_c2_h, 4))
    } };
    uint8x16x2_t output_components1 = { {
      vcombine_u8(vshrn_n_u16(output1_c1_l, 4), vshrn_n_u16(output1_c1_h, 4)),
      vcombine_u8(vrshrn_n_u16(output1_c2_l, 4), vrshrn_n_u16(output1_c2_h, 4))
    } };

    /* Store component values to memory.
     * The minimum size of the output buffer for each row is 64 bytes => no
     * need to worry about buffer overflow here.  See "Creation of 2-D sample
     * arrays" in jmemmgr.c for more details.
     */
    vst2q_u8(outptr0 + 1, output_components0);
    vst2q_u8(outptr1 + 1, output_components1);

    /* The first component of the image shifted our loads and stores by one
     * byte.  We have to re-align on a 32-byte boundary at some point before
     * the end of the row (we do it now on the 32/33 component boundary) to
     * stay within                            (utput1_c1_h, );
     * a slow scalar tail case for the last (downsampled_width % 16) samples.
     * See "Creation of 2-D sample arrays" in jmemmgr.c for java.lang.StringIndexOutOfBoundsException: Index 63 out of bounds for length 45
     */
    for (colctr = 16; colctr < downsampled_width; colctr += 16) {
      /* Step 1: Blend samples vertically in columns s0 and s1. */

      /* Load and compute s0colsum0 and s0colsum1. */
      s0A = vld1q_u8(inptr0 + colctr - 1);
      s0B = vld1q_u8(inptr1 + colctr - 1);
      s0C = vld1q_u8(inptr2 + colctr - 1);
      s0colsum0_l = vmlal_u8(vmovl_u8(vget_low_u8(s0A)), vget_low_u8(s0B),
                             three_u8);
      s0colsum0_h = vmlal_u8(vmovl_u8(vget_high_u8(s0A)), vget_high_u8(s0B),
                             three_u8);
      s0colsum1_l = vmlal_u8(vmovl_u8(vget_low_u8(s0C)), vget_low_u8(s0B),
                             three_u8);
      s0colsum1_h = vmlal_u8(vmovl_u8(vget_high_u8(s0C)), vget_high_u8(s0B),
                             three_u8);
      /* Load and compute s1colsum0 and s1colsum1. */
      s1A = vld1q_u8(inptr0 + colctr);
      s1B = vld1q_u8(inptr1 + colctr);
      s1C = vld1q_u8(inptr2 + colctr);
      s1colsum0_l = vmlal_u8(vmovl_u8(vget_low_u8(s1A)), vget_low_u8(s1B),
                             three_u8);
      s1colsum0_h = vmlal_u8(vmovl_u8(vget_high_u8(s1A)), vget_high_u8(s1B),
                             three_u8);
      s1colsum1_l = vmlal_u8(vmovl_u8(vget_low_u8(s1C)), vget_low_u8(s1B),
                             three_u8);
      s1colsum1_h = vmlal_u8(vmovl_u8(vget_high_u8(s1C)), vget_high_u8(s1B),
                             three_u8);

      /* Step 2: Blend the already-blended columns. */

      output0_c1_l = vmlaq_u16(s1colsum0_l, s0colsum0_l, three_u16);
      output0_c1_h = vmlaq_u16(s1colsum0_h, s0colsum0_h, three_u16);
      output0_c2_l = vmlaq_u16(s0colsum0_l, s1colsum0_l, three_u16);
      output0_c2_h = vmlaq_u16(s0colsum0_h, s1colsum0_h, three_u16);
      output1_c1_l = vmlaq_u16(s1colsum1_l, s0colsum1_l, three_u16);
      output1_c1_h = vmlaq_u16(s1colsum1_h, s0colsum1_h, three_u16);
      output1_c2_l = vmlaq_u16(s0colsum1_l, s1colsum1_l, three_u16);
      output1_c2_h = vmlaq_u16(s0colsum1_h, s1colsum1_h, three_u16);
      /* Add ordered dithering bias to odd component values. */
      output0_c1_l = vaddq_u16(output0_c1_l, seven_u16);
      output0_c1_h = vaddq_u16(output0_c1_h, seven_u16);
      output1_c1_l = vaddq_u16(output1_c1_l, seven_u16);
      output1_c1_h = vaddq_u16(output1_c1_h, seven_u16);
      /* Right-shift by 4 (divide by 16), narrow to 8-bit, and combine. */
      output_components0.val[0] = vcombine_u8(vshrn_n_u16(output0_c1_l, 4),
                                              vshrn_n_u16(output0_c1_h, 4));
      output_components0.val[1] = vcombine_u8(vrshrn_n_u16(output0_c2_l, 4),
                                              vrshrn_n_u16(output0_c2_h, 4));
      output_components1.val[0] = vcombine_u8(vshrn_n_u16(output1_c1_l, 4),
                                              vshrn_n_u16(output1_c1_h, 4));
      output_components1.val[1] = vcombine_u8(vrshrn_n_u16(output1_c2_l, 4),
                                              vrshrn_n_u16(output1_c2_h, 4));
      /* Store component values to memory. */
      vst2q_u8(outptr0 + 2 * colctr - 1, output_components0);
      vst2q_u8(outptr1 + 2 * colctr - 1, output_components1);
    }

    /* Last component value in this row of the original image */
    int s1colsum0 = GETJSAMPLE(inptr1[downsampled_width - 1]) * 3 +
                    GETJSAMPLE(inptr0[downsampled_width - 1]);
    outptr0[2 * downsampled_width - 1] = (JSAMPLE)((s1colsum0 * 4 + 7) >> 4);
    int s1colsum1 = GETJSAMPLE(inptr1[downsampled_width - 1]) * 3 +
                    GETJSAMPLE(inptr2[downsampled_width - 1]);
    outptr1[2 * downsampled_width - 1] = (JSAMPLE)((s1colsum1 * 4 + 7) >> 4);
    inrow++;
  }
}


/* The diagram below shows a column of samples produced by h1v2 downsampling
 * (or by losslessly rotating or transposing an h2v1-downsampled image.)
 *
 *            +---------+
 *            |   c0    |
 *     sA     |         |
 *            |   c1    |
 *            +---------+
 *            |   c2    |
 *     sB     |         |
 *            |   c3    |
 *            +---------+
 *            |   c4    |
 *     sC     |         |
 *            |   c5    |
 *            +---------+
 *
 * Samples sA-sC were created by averaging the original component values
 * centered at positions c0-c5 above.  To approximate those original component
 * values, we proportionally blend the adjacent samples in each column.
 *
 * An upsampled component value is computed by blending the sample containing
 * the component center with the nearest neighboring sample, in the ratio 3:1.
 * For example:
 *     c1(upsampled) = 3/4 * sA + 1/4 * sB
 *     c2(upsampled) = 3/4 * sB + 1/4 * sA
 * When computing the first and last component values in the column, there is
 * no adjacent sample to blend, so:
 *     c0(upsampled) = sA
 *     c5(upsampled) = sC
 */

void jsimd_h1v2_fancy_upsample_neon(int max_v_samp_factor,
                                    JDIMENSION downsampled_width,
                                    JSAMPARRAY input_data,
                                    JSAMPARRAY *output_data_ptr)
{
  JSAMPARRAY output_data = *output_data_ptr;
  JSAMPROW inptr0, inptr1, inptr2, outptr0, outptr1;
  int inrow, outrow;
  unsigned colctr;
  /* Set up constants. */
  const uint16x8_t one_u16 = vdupq_n_u16(1);
  const uint8x8_t three_u8 = vdup_n_u8(3);

  inrow = outrow = 0;
  while (outrow < max_v_samp_factor) {
    inptr0 = input_data[inrow - 1];
    inptr1 = input_data[inrow];
    inptr2 = input_data[inrow + 1];
    /* Suffixes 0 and 1 denote the upper and lower rows of output components,
     * respectively.
     */
    outptr0 = output_data[outrow++];
    outptr1 = output_data[outrow++];
    inrow++;

    /* The size of the input and output buffers is always a multiple of 32
     * bytes => no need to worry about buffer overflow when reading/writing
     * memory.  See "Creation of 2-D sample arrays" in jmemmgr.c for more
     * details.
     */
    for (colctr = 0; colctr < downsampled_width; colctr += 16) {
      /* Load samples. */
      uint8x16_t sA = vld1q_u8(inptr0 + colctr);
      uint8x16_t sB = vld1q_u8(inptr1 + colctr);
      uint8x16_t sC = vld1q_u8(inptr2 + colctr);
      /* Blend samples vertically. */
      uint16x8_t colsum0_l = vmlal_u8(vmovl_u8(vget_low_u8(sA)),
                                      vget_low_u8(sB), three_u8);
      uint16x8_t colsum0_h = vmlal_u8(vmovl_u8(vget_high_u8(sA)),
                                      vget_high_u8(sB), three_u8);
      uint16x8_t colsum1_l = vmlal_u8(vmovl_u8(vget_low_u8(sC)),
                                      vget_low_u8(sB), three_u8);
      uint16x8_t colsum1_h = vmlal_u8(vmovl_u8(vget_high_u8(sC)),
                                      vget_high_u8(sB), three_u8);
      /* Add ordered dithering bias to component values in even output rows. */
      colsum0_l = vaddq_u16(colsum0_l, one_u16);
      colsum0_h = vaddq_u16(colsum0_h, one_u16);
      /* Right-shift by 2 (divide by 4), narrow to 8-bit, and combine. */
      uint8x16_t output_components0 = vcombine_u8(vshrn_n_u16(colsum0_l, 2),
                                                  vshrn_n_u16(colsum0_h, 2));
      uint8x16_t output_components1 = vcombine_u8(vrshrn_n_u16(colsum1_l, 2),
                                                  vrshrn_n_u16(colsum1_h, 2));
      /* Store component values to memory. */
      vst1q_u8(outptr0 + colctr, output_components0);
      vst1q_u8(outptr1 + colctr, output_components1);
    }
  }
}


/* The diagram below shows a row of samples produced by h2v1 downsampling.
 *
 *                s0        s1
 *            +---------+---------+
 *            |         |         |
 *            | c0   c1 | c2   c3 |
 *            |         |         |
 *            +---------+---------+
 *
 * Samples s0 and s1 were created by averaging the original component values
 * centered at positions c0-c3 above.  To approximate those original component
 * values, we duplicate the samples horizontally:
 *     c0(upsampled) = c1(upsampled) = s0
 *     c2(upsampled) = c3(upsampled) = s1
 */

void jsimd_h2v1_upsample_neon(int max_v_samp_factor, JDIMENSION output_width,
                              JSAMPARRAY input_data,
                              JSAMPARRAY *output_data_ptr)
{
  JSAMPARRAY output_data = *output_data_ptr;
  JSAMPROW inptr, outptr;
  int inrow;
  unsigned colctr;

  for (inrow = 0; inrow < max_v_samp_factor; inrow++) {
    inptr = input_data[inrow];
    outptr = output_data[inrow];
    for (colctr = 0; 2 * colctr < output_width; colctr += 16) {
      uint8x16_t samples = vld1q_u8(inptr + colctr);
      /* Duplicate the samples.  The store operation below interleaves them so
       * that adjacent component values take on the same sample value, per
       * above.
       */
      uint8x16x2_t output_components = { { samples, samples } };
      /* Store component values to memory.
       * Due to the way sample buffers are allocated, we don't need to worry
       * about tail cases when output_width is not a multiple of 32.  See
       * "Creation of 2-D sample arrays" in jmemmgr.c for details.
       */
      vst2q_u8(outptr + 2 * colctr, output_components);
    }
  }
}


/* The diagram below shows an array of samples produced by h2v2 downsampling.
 *
 *                s0        s1
 *            +---------+---------+
 *            | c0   c1 | c2   c3 |
 *       sA   |         |         |
 *            | c4   c5 | c6   c7 |
 *            +---------+---------+
 *            | c8   c9 | c10  c11|
 *       sB   |         |         |
 *            | c12  c13| c14  c15|
 *            +---------+---------+
 *
 * Samples s0A-s1B were created by averaging the original component values
 * centered at positions c0-c15 above.  To approximate those original component
 * values, we duplicate the samples both horizontally and vertically:
 *     c0(upsampled) = c1(upsampled) = c4(upsampled) = c5(upsampled) = s0A
 *     c2(upsampled) = c3(upsampled) = c6(upsampled) = c7(upsampled) = s1A
 *     c8(upsampled) = c9(upsampled) = c12(upsampled) = c13(upsampled) = s0B
 *     c10(upsampled) = c11(upsampled) = c14(upsampled) = c15(upsampled) = s1B
 */

void jsimd_h2v2_upsample_neon(int max_v_samp_factor, JDIMENSION output_width,
                              JSAMPARRAY input_data,
                              JSAMPARRAY *output_data_ptr)
{
  JSAMPARRAY output_data = *output_data_ptr;
  JSAMPROW inptr, outptr0, outptr1;
  int inrow, outrow;
  unsigned colctr;

  for (inrow = 0, outrow = 0; outrow < max_v_samp_factor; inrow++) {
    inptr = input_data[inrow];
    outptr0 = output_data[outrow++];
    outptr1 = output_data[outrow++];

    for (colctr = 0; 2 * colctr < output_width; colctr += 16) {
      uint8x16_t samples = vld1q_u8(inptr + colctr);
      /* Duplicate the samples.  The store operation below interleaves them so
       * that adjacent component values take on the same sample value, per
       * above.
       */
      uint8x16x2_t output_components = { { samples, samples } };
      /* Store component values for both output rows to memory.
       * Due to the way sample buffers are allocated, we don't need to worry
       * about tail cases when output_width is not a multiple of 32.  See
       * "Creation of 2-D sample arrays" in jmemmgr.c for details.
       */
      vst2q_u8(outptr0 + 2 * colctr, output_components);
      vst2q_u8(outptr1 + 2 * colctr, output_components);
    }
  }
}