Quelle c2_MacroAssembler_s390.cpp Sprache: C

/*
* Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2017, 2022 SAP SE. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*
*/

#include "precompiled.hpp"
#include "asm/assembler.hpp"
#include "asm/assembler.inline.hpp"
#include "opto/c2_MacroAssembler.hpp"
#include "opto/intrinsicnode.hpp"
#include "runtime/stubRoutines.hpp"

#define BLOCK_COMMENT(str) block_comment(str)
#define BIND(label)        bind(label); BLOCK_COMMENT(#label ":")

//------------------------------------------------------
//   Special String Intrinsics. Implementation
//------------------------------------------------------

// Intrinsics for CompactStrings

// Compress char[] to byte[].
//   Restores: src, dst
//   Uses:     cnt
//   Kills:    tmp, Z_R0, Z_R1.
//   Early clobber: result.
// Note:
//   cnt is signed int. Do not rely on high word!
//       counts # characters, not bytes.
//
// The result indicates success or failure of the operation.
//   General compress operation (cut off high order byte which must be all zeroes).
//    = len - all characters have been successfully compressed.
//    = 0   - compress failed. At least one character was found with a non-zero high order byte.
//            This is the failure return value which exactly corresponds to the Java implementation.
//    0 <= result < len - compress failed. That many characters were compressed successfully
//                        before the first non-compressable character was found. This is the
//                        current, but not fully compatible, implementation. See below.
//   Encode to ISO or 7-bit ASCII array.
//    = len - all characters have been encoded successfully.
//    < len - encode failed. That many characters were encoded successfully.
//            When used as an index into the character array, the return value addresses the
//            first not encodeable character.
//
// If precise is true, the processing stops exactly at the point where a failure is detected.
// More characters than indicated by the return value may have been read from the src array.
// Exactly the number of characters indicated by the return value have been written to dst.
// If precise is false, a few characters more than indicated by the return value may have been
// written to the dst array. In any failure case, The result value indexes the first invalid character.
unsigned int C2_MacroAssembler::string_compress(Register result, Register src, Register dst, Register cnt,
                                                Register tmp,    bool precise, bool toASCII) {
  assert_different_registers(Z_R0, Z_R1, result, src, dst, cnt, tmp);

  unsigned short char_mask = 0xff00;  // all selected bits must be '0' for a char to be valid
  unsigned int   mask_ix_l = 0;       // leftmost one bit pos in mask
  unsigned int   mask_ix_r = 7;       // rightmost one bit pos in mask
  if (precise) {
    if (toASCII) {
      BLOCK_COMMENT("encode_ascii_array {");
      char_mask = 0xff80;
      mask_ix_r = 8;         // rightmost one bit pos in mask. ASCII only uses codes 0..127
    } else {
      BLOCK_COMMENT("encode_iso_array {");
    }
  } else {
    BLOCK_COMMENT("string_compress {");
    assert(!toASCII, "Can't compress strings to 7-bit ASCII");
  }
  int  block_start = offset();

  Register       Rsrc  = src;
  Register       Rdst  = dst;
  Register       Rix   = tmp;
  Register       Rcnt  = cnt;
  Register       Rmask = result;  // holds incompatibility check mask until result value is stored.
  Label          ScalarShortcut, AllDone;

  z_iilf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask);
  z_iihf(Rmask, (unsigned int)char_mask<<16 | (unsigned int)char_mask);

#if 0  // Sacrifice shortcuts for code compactness
  {
    //---<  shortcuts for short strings (very frequent)   >---
    //   Strings with 4 and 8 characters were found to occur very frequently.
    //   Therefore, we handle them right away with minimal overhead.
    Label     skipShortcut, skip4Shortcut, skip8Shortcut;
    Register  Rout = Z_R0;
    z_chi(Rcnt, 4);
    z_brne(skip4Shortcut);                 // 4 characters are very frequent
      z_lg(Z_R0, 0, Rsrc);                 // Treat exactly 4 characters specially.
      if (VM_Version::has_DistinctOpnds()) {
        Rout = Z_R0;
        z_ngrk(Rix, Z_R0, Rmask);
      } else {
        Rout = Rix;
        z_lgr(Rix, Z_R0);
        z_ngr(Z_R0, Rmask);
      }
      z_brnz(skipShortcut);
      z_stcmh(Rout, 5, 0, Rdst);
      z_stcm(Rout,  5, 2, Rdst);
      z_lgfr(result, Rcnt);
      z_bru(AllDone);
    bind(skip4Shortcut);

    z_chi(Rcnt, 8);
    z_brne(skip8Shortcut);                 // There's more to do...
      z_lmg(Z_R0, Z_R1, 0, Rsrc);          // Treat exactly 8 characters specially.
      if (VM_Version::has_DistinctOpnds()) {
        Rout = Z_R0;
        z_ogrk(Rix, Z_R0, Z_R1);
        z_ngr(Rix, Rmask);
      } else {
        Rout = Rix;
        z_lgr(Rix, Z_R0);
        z_ogr(Z_R0, Z_R1);
        z_ngr(Z_R0, Rmask);
      }
      z_brnz(skipShortcut);
      z_stcmh(Rout, 5, 0, Rdst);
      z_stcm(Rout,  5, 2, Rdst);
      z_stcmh(Z_R1, 5, 4, Rdst);
      z_stcm(Z_R1,  5, 6, Rdst);
      z_lgfr(result, Rcnt);
      z_bru(AllDone);

    bind(skip8Shortcut);
    clear_reg(Z_R0, true, false);          // #characters already processed (none). Precond for scalar loop.
    z_brl(ScalarShortcut);                 // Just a few characters

    bind(skipShortcut);
  }
#endif
  clear_reg(Z_R0);                         // make sure register is properly initialized.

  if (VM_Version::has_VectorFacility()) {
    const int  min_vcnt     = 32;          // Minimum #characters required to use vector instructions.
                                           // Otherwise just do nothing in vector mode.
                                           // Must correspond to # vector registers used by implementation,
                                           // and must be a power of 2.
    const int  log_min_vcnt = exact_log2(min_vcnt);
    Label      VectorLoop, VectorDone, VectorBreak;

    VectorRegister Vtmp1      = Z_V16;
    VectorRegister Vtmp2      = Z_V17;
    VectorRegister Vmask      = Z_V18;
    VectorRegister Vzero      = Z_V19;
    VectorRegister Vsrc_first = Z_V20;
    VectorRegister Vsrc_last  = Z_V23;

    assert((Vsrc_last->encoding() - Vsrc_first->encoding() + 1) == min_vcnt/8, "logic error");
    assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
    z_srak(Rix, Rcnt, log_min_vcnt);       // # vector loop iterations
    z_brz(VectorDone);                     // not enough data for vector loop

    z_vzero(Vzero);                        // all zeroes
    z_vgmh(Vmask, mask_ix_l, mask_ix_r);   // generate 0xff00/0xff80 mask for all 2-byte elements
    z_sllg(Z_R0, Rix, log_min_vcnt);       // remember #chars that will be processed by vector loop

    bind(VectorLoop);
      z_vlm(Vsrc_first, Vsrc_last, 0, Rsrc);
      add2reg(Rsrc, min_vcnt*2);

      //---<  check for incompatible character  >---
      z_vo(Vtmp1, Z_V20, Z_V21);
      z_vo(Vtmp2, Z_V22, Z_V23);
      z_vo(Vtmp1, Vtmp1, Vtmp2);
      z_vn(Vtmp1, Vtmp1, Vmask);
      z_vceqhs(Vtmp1, Vtmp1, Vzero);       // all bits selected by mask must be zero for successful compress.
      z_bvnt(VectorBreak);                 // break vector loop if not all vector elements compare eq -> incompatible character found.
                                           // re-process data from current iteration in break handler.

      //---<  pack & store characters  >---
      z_vpkh(Vtmp1, Z_V20, Z_V21);         // pack (src1, src2) -> tmp1
      z_vpkh(Vtmp2, Z_V22, Z_V23);         // pack (src3, src4) -> tmp2
      z_vstm(Vtmp1, Vtmp2, 0, Rdst);       // store packed string
      add2reg(Rdst, min_vcnt);

      z_brct(Rix, VectorLoop);

    z_bru(VectorDone);

    bind(VectorBreak);
      add2reg(Rsrc, -min_vcnt*2);          // Fix Rsrc. Rsrc was already updated, but Rdst and Rix are not.
      z_sll(Rix, log_min_vcnt);            // # chars processed so far in VectorLoop, excl. current iteration.
      z_sr(Z_R0, Rix);                     // correct # chars processed in total.

    bind(VectorDone);
  }

  {
    const int  min_cnt     =  8;           // Minimum #characters required to use unrolled loop.
                                           // Otherwise just do nothing in unrolled loop.
                                           // Must correspond to # registers used by implementation,
                                           // and must be a power of 2.
    const int  log_min_cnt = exact_log2(min_cnt);
    Label      UnrolledLoop, UnrolledDone, UnrolledBreak;

    if (VM_Version::has_DistinctOpnds()) {
      z_srk(Rix, Rcnt, Z_R0);              // remaining # chars to compress in unrolled loop
    } else {
      z_lr(Rix, Rcnt);
      z_sr(Rix, Z_R0);
    }
    z_sra(Rix, log_min_cnt);               // unrolled loop count
    z_brz(UnrolledDone);

    bind(UnrolledLoop);
      z_lmg(Z_R0, Z_R1, 0, Rsrc);
      if (precise) {
        z_ogr(Z_R1, Z_R0);                 // check all 8 chars for incompatibility
        z_ngr(Z_R1, Rmask);
        z_brnz(UnrolledBreak);

        z_lg(Z_R1, 8, Rsrc);               // reload destroyed register
        z_stcmh(Z_R0, 5, 0, Rdst);
        z_stcm(Z_R0,  5, 2, Rdst);
      } else {
        z_stcmh(Z_R0, 5, 0, Rdst);
        z_stcm(Z_R0,  5, 2, Rdst);

        z_ogr(Z_R0, Z_R1);
        z_ngr(Z_R0, Rmask);
        z_brnz(UnrolledBreak);
      }
      z_stcmh(Z_R1, 5, 4, Rdst);
      z_stcm(Z_R1,  5, 6, Rdst);

      add2reg(Rsrc, min_cnt*2);
      add2reg(Rdst, min_cnt);
      z_brct(Rix, UnrolledLoop);

    z_lgfr(Z_R0, Rcnt);                    // # chars processed in total after unrolled loop.
    z_nilf(Z_R0, ~(min_cnt-1));
    z_tmll(Rcnt, min_cnt-1);
    z_brnaz(ScalarShortcut);               // if all bits zero, there is nothing left to do for scalar loop.
                                           // Rix == 0 in all cases.
    z_sllg(Z_R1, Rcnt, 1);                 // # src bytes already processed. Only lower 32 bits are valid!
                                           //   Z_R1 contents must be treated as unsigned operand! For huge strings,
                                           //   (Rcnt >= 2**30), the value may spill into the sign bit by sllg.
    z_lgfr(result, Rcnt);                  // all characters processed.
    z_slgfr(Rdst, Rcnt);                   // restore ptr
    z_slgfr(Rsrc, Z_R1);                   // restore ptr, double the element count for Rsrc restore
    z_bru(AllDone);

    bind(UnrolledBreak);
    z_lgfr(Z_R0, Rcnt);                    // # chars processed in total after unrolled loop
    z_nilf(Z_R0, ~(min_cnt-1));
    z_sll(Rix, log_min_cnt);               // # chars not yet processed in UnrolledLoop (due to break), broken iteration not included.
    z_sr(Z_R0, Rix);                       // fix # chars processed OK so far.
    if (!precise) {
      // Because we don't need to be precise, we just return the # of characters which have been written.
      // The first illegal character is in the index range [result-min_cnt/2, result+min_cnt/2).
      z_lgfr(result, Z_R0);
      z_sllg(Z_R1, Z_R0, 1);               // # src bytes already processed. Only lower 32 bits are valid!
                                           //   Z_R1 contents must be treated as unsigned operand! For huge strings,
                                           //   (Rcnt >= 2**30), the value may spill into the sign bit by sllg.
      z_aghi(result, min_cnt/2);           // min_cnt/2 characters have already been written
                                           // but ptrs were not updated yet.
      z_slgfr(Rdst, Z_R0);                 // restore ptr
      z_slgfr(Rsrc, Z_R1);                 // restore ptr, double the element count for Rsrc restore
      z_bru(AllDone);
    }
    bind(UnrolledDone);
  }

  {
    Label     ScalarLoop, ScalarDone, ScalarBreak;

    bind(ScalarShortcut);
    z_ltgfr(result, Rcnt);
    z_brz(AllDone);

#if 0  // Sacrifice shortcuts for code compactness
    {
      //---<  Special treatment for very short strings (one or two characters)  >---
      //   For these strings, we are sure that the above code was skipped.
      //   Thus, no registers were modified, register restore is not required.
      Label     ScalarDoit, Scalar2Char;
      z_chi(Rcnt, 2);
      z_brh(ScalarDoit);
      z_llh(Z_R1,  0, Z_R0, Rsrc);
      z_bre(Scalar2Char);
      z_tmll(Z_R1, char_mask);
      z_lghi(result, 0);                   // cnt == 1, first char invalid, no chars successfully processed
      z_brnaz(AllDone);
      z_stc(Z_R1,  0, Z_R0, Rdst);
      z_lghi(result, 1);
      z_bru(AllDone);

      bind(Scalar2Char);
      z_llh(Z_R0,  2, Z_R0, Rsrc);
      z_tmll(Z_R1, char_mask);
      z_lghi(result, 0);                   // cnt == 2, first char invalid, no chars successfully processed
      z_brnaz(AllDone);
      z_stc(Z_R1,  0, Z_R0, Rdst);
      z_tmll(Z_R0, char_mask);
      z_lghi(result, 1);                   // cnt == 2, second char invalid, one char successfully processed
      z_brnaz(AllDone);
      z_stc(Z_R0,  1, Z_R0, Rdst);
      z_lghi(result, 2);
      z_bru(AllDone);

      bind(ScalarDoit);
    }
#endif

    if (VM_Version::has_DistinctOpnds()) {
      z_srk(Rix, Rcnt, Z_R0);              // remaining # chars to compress in scalar loop
    } else {
      z_lr(Rix, Rcnt);
      z_sr(Rix, Z_R0);
    }
    z_lgfr(result, Rcnt);                  // # processed characters (if all encodes ok).
    z_brz(ScalarDone);                     // anything left to do? (uses CC from Rix calculation)

    bind(ScalarLoop);
      z_llh(Z_R1, 0, Z_R0, Rsrc);
      z_tmll(Z_R1, char_mask);
      z_brnaz(ScalarBreak);
      z_stc(Z_R1, 0, Z_R0, Rdst);
      add2reg(Rsrc, 2);
      add2reg(Rdst, 1);
      z_brct(Rix, ScalarLoop);

    z_bru(ScalarDone);

    bind(ScalarBreak);
    z_sr(result, Rix);

    bind(ScalarDone);
    z_sgfr(Rdst, result);                  // restore ptr
    z_sgfr(Rsrc, result);                  // restore ptr, double the element count for Rsrc restore
    z_sgfr(Rsrc, result);
  }
  bind(AllDone);

  if (precise) {
    if (toASCII) {
      BLOCK_COMMENT("} encode_ascii_array");
    } else {
      BLOCK_COMMENT("} encode_iso_array");
    }
  } else {
    BLOCK_COMMENT("} string_compress");
  }
  return offset() - block_start;
}

// Inflate byte[] to char[].
unsigned int C2_MacroAssembler::string_inflate_trot(Register src, Register dst, Register cnt, Register tmp) {
  int block_start = offset();

  BLOCK_COMMENT("string_inflate {");

  Register stop_char = Z_R0;
  Register table     = Z_R1;
  Register src_addr  = tmp;

  assert_different_registers(Z_R0, Z_R1, tmp, src, dst, cnt);
  assert(dst->encoding()%2 == 0, "must be even reg");
  assert(cnt->encoding()%2 == 1, "must be odd reg");
  assert(cnt->encoding() - dst->encoding() == 1, "must be even/odd pair");

  StubRoutines::zarch::generate_load_trot_table_addr(this, table);  // kills Z_R0 (if ASSERT)
  clear_reg(stop_char);  // Stop character. Not used here, but initialized to have a defined value.
  lgr_if_needed(src_addr, src);
  z_llgfr(cnt, cnt);     // # src characters, must be a positive simm32.

  translate_ot(dst, src_addr, /* mask = */ 0x0001);

  BLOCK_COMMENT("} string_inflate");

  return offset() - block_start;
}

// Inflate byte[] to char[].
//   Restores: src, dst
//   Uses:     cnt
//   Kills:    tmp, Z_R0, Z_R1.
// Note:
//   cnt is signed int. Do not rely on high word!
//       counts # characters, not bytes.
unsigned int C2_MacroAssembler::string_inflate(Register src, Register dst, Register cnt, Register tmp) {
  assert_different_registers(Z_R0, Z_R1, src, dst, cnt, tmp);

  BLOCK_COMMENT("string_inflate {");
  int block_start = offset();

  Register   Rcnt = cnt;   // # characters (src: bytes, dst: char (2-byte)), remaining after current loop.
  Register   Rix  = tmp;   // loop index
  Register   Rsrc = src;   // addr(src array)
  Register   Rdst = dst;   // addr(dst array)
  Label      ScalarShortcut, AllDone;

#if 0  // Sacrifice shortcuts for code compactness
  {
    //---<  shortcuts for short strings (very frequent)   >---
    Label   skipShortcut, skip4Shortcut;
    z_ltr(Rcnt, Rcnt);                     // absolutely nothing to do for strings of len == 0.
    z_brz(AllDone);
    clear_reg(Z_R0);                       // make sure registers are properly initialized.
    clear_reg(Z_R1);
    z_chi(Rcnt, 4);
    z_brne(skip4Shortcut);                 // 4 characters are very frequent
      z_icm(Z_R0, 5,    0, Rsrc);          // Treat exactly 4 characters specially.
      z_icm(Z_R1, 5,    2, Rsrc);
      z_stm(Z_R0, Z_R1, 0, Rdst);
      z_bru(AllDone);
    bind(skip4Shortcut);

    z_chi(Rcnt, 8);
    z_brh(skipShortcut);                   // There's a lot to do...
    z_lgfr(Z_R0, Rcnt);                    // remaining #characters (<= 8). Precond for scalar loop.
                                           // This does not destroy the "register cleared" state of Z_R0.
    z_brl(ScalarShortcut);                 // Just a few characters
      z_icmh(Z_R0, 5, 0, Rsrc);            // Treat exactly 8 characters specially.
      z_icmh(Z_R1, 5, 4, Rsrc);
      z_icm(Z_R0,  5, 2, Rsrc);
      z_icm(Z_R1,  5, 6, Rsrc);
      z_stmg(Z_R0, Z_R1, 0, Rdst);
      z_bru(AllDone);
    bind(skipShortcut);
  }
#endif
  clear_reg(Z_R0);                         // make sure register is properly initialized.

  if (VM_Version::has_VectorFacility()) {
    const int  min_vcnt     = 32;          // Minimum #characters required to use vector instructions.
                                           // Otherwise just do nothing in vector mode.
                                           // Must be multiple of vector register length (16 bytes = 128 bits).
    const int  log_min_vcnt = exact_log2(min_vcnt);
    Label      VectorLoop, VectorDone;

    assert(VM_Version::has_DistinctOpnds(), "Assumption when has_VectorFacility()");
    z_srak(Rix, Rcnt, log_min_vcnt);       // calculate # vector loop iterations
    z_brz(VectorDone);                     // skip if none

    z_sllg(Z_R0, Rix, log_min_vcnt);       // remember #chars that will be processed by vector loop

    bind(VectorLoop);
      z_vlm(Z_V20, Z_V21, 0, Rsrc);        // get next 32 characters (single-byte)
      add2reg(Rsrc, min_vcnt);

      z_vuplhb(Z_V22, Z_V20);              // V2 <- (expand) V0(high)
      z_vupllb(Z_V23, Z_V20);              // V3 <- (expand) V0(low)
      z_vuplhb(Z_V24, Z_V21);              // V4 <- (expand) V1(high)
      z_vupllb(Z_V25, Z_V21);              // V5 <- (expand) V1(low)
      z_vstm(Z_V22, Z_V25, 0, Rdst);       // store next 32 bytes
      add2reg(Rdst, min_vcnt*2);

      z_brct(Rix, VectorLoop);

    bind(VectorDone);
  }

  const int  min_cnt     =  8;             // Minimum #characters required to use unrolled scalar loop.
                                           // Otherwise just do nothing in unrolled scalar mode.
                                           // Must be multiple of 8.
  {
    const int  log_min_cnt = exact_log2(min_cnt);
    Label      UnrolledLoop, UnrolledDone;

    if (VM_Version::has_DistinctOpnds()) {
      z_srk(Rix, Rcnt, Z_R0);              // remaining # chars to process in unrolled loop
    } else {
      z_lr(Rix, Rcnt);
      z_sr(Rix, Z_R0);
    }
    z_sra(Rix, log_min_cnt);               // unrolled loop count
    z_brz(UnrolledDone);

    clear_reg(Z_R0);
    clear_reg(Z_R1);

    bind(UnrolledLoop);
      z_icmh(Z_R0, 5, 0, Rsrc);
      z_icmh(Z_R1, 5, 4, Rsrc);
      z_icm(Z_R0,  5, 2, Rsrc);
      z_icm(Z_R1,  5, 6, Rsrc);
      add2reg(Rsrc, min_cnt);

      z_stmg(Z_R0, Z_R1, 0, Rdst);

      add2reg(Rdst, min_cnt*2);
      z_brct(Rix, UnrolledLoop);

    bind(UnrolledDone);
    z_lgfr(Z_R0, Rcnt);                    // # chars left over after unrolled loop.
    z_nilf(Z_R0, min_cnt-1);
    z_brnz(ScalarShortcut);                // if zero, there is nothing left to do for scalar loop.
                                           // Rix == 0 in all cases.
    z_sgfr(Z_R0, Rcnt);                    // negative # characters the ptrs have been advanced previously.
    z_agr(Rdst, Z_R0);                     // restore ptr, double the element count for Rdst restore.
    z_agr(Rdst, Z_R0);
    z_agr(Rsrc, Z_R0);                     // restore ptr.
    z_bru(AllDone);
  }

  {
    bind(ScalarShortcut);
    // Z_R0 must contain remaining # characters as 64-bit signed int here.
    //      register contents is preserved over scalar processing (for register fixup).

#if 0  // Sacrifice shortcuts for code compactness
    {
      Label      ScalarDefault;
      z_chi(Rcnt, 2);
      z_brh(ScalarDefault);
      z_llc(Z_R0,  0, Z_R0, Rsrc);     // 6 bytes
      z_sth(Z_R0,  0, Z_R0, Rdst);     // 4 bytes
      z_brl(AllDone);
      z_llc(Z_R0,  1, Z_R0, Rsrc);     // 6 bytes
      z_sth(Z_R0,  2, Z_R0, Rdst);     // 4 bytes
      z_bru(AllDone);
      bind(ScalarDefault);
    }
#endif

    Label   CodeTable;
    // Some comments on Rix calculation:
    //  - Rcnt is small, therefore no bits shifted out of low word (sll(g) instructions).
    //  - high word of both Rix and Rcnt may contain garbage
    //  - the final lngfr takes care of that garbage, extending the sign to high word
    z_sllg(Rix, Z_R0, 2);                // calculate 10*Rix = (4*Rix + Rix)*2
    z_ar(Rix, Z_R0);
    z_larl(Z_R1, CodeTable);
    z_sll(Rix, 1);
    z_lngfr(Rix, Rix);      // ix range: [0..7], after inversion & mult: [-(7*12)..(0*12)].
    z_bc(Assembler::bcondAlways, 0, Rix, Z_R1);

    z_llc(Z_R1,  6, Z_R0, Rsrc);  // 6 bytes
    z_sth(Z_R1, 12, Z_R0, Rdst);  // 4 bytes

    z_llc(Z_R1,  5, Z_R0, Rsrc);
    z_sth(Z_R1, 10, Z_R0, Rdst);

    z_llc(Z_R1,  4, Z_R0, Rsrc);
    z_sth(Z_R1,  8, Z_R0, Rdst);

    z_llc(Z_R1,  3, Z_R0, Rsrc);
    z_sth(Z_R1,  6, Z_R0, Rdst);

    z_llc(Z_R1,  2, Z_R0, Rsrc);
    z_sth(Z_R1,  4, Z_R0, Rdst);

    z_llc(Z_R1,  1, Z_R0, Rsrc);
    z_sth(Z_R1,  2, Z_R0, Rdst);

    z_llc(Z_R1,  0, Z_R0, Rsrc);
    z_sth(Z_R1,  0, Z_R0, Rdst);
    bind(CodeTable);

    z_chi(Rcnt, 8);                        // no fixup for small strings. Rdst, Rsrc were not modified.
    z_brl(AllDone);

    z_sgfr(Z_R0, Rcnt);                    // # characters the ptrs have been advanced previously.
    z_agr(Rdst, Z_R0);                     // restore ptr, double the element count for Rdst restore.
    z_agr(Rdst, Z_R0);
    z_agr(Rsrc, Z_R0);                     // restore ptr.
  }
  bind(AllDone);

  BLOCK_COMMENT("} string_inflate");
  return offset() - block_start;
}

// Inflate byte[] to char[], length known at compile time.
//   Restores: src, dst
//   Kills:    tmp, Z_R0, Z_R1.
// Note:
//   len is signed int. Counts # characters, not bytes.
unsigned int C2_MacroAssembler::string_inflate_const(Register src, Register dst, Register tmp, int len) {
  assert_different_registers(Z_R0, Z_R1, src, dst, tmp);

  BLOCK_COMMENT("string_inflate_const {");
  int block_start = offset();

  Register   Rix  = tmp;   // loop index
  Register   Rsrc = src;   // addr(src array)
  Register   Rdst = dst;   // addr(dst array)
  Label      ScalarShortcut, AllDone;
  int        nprocessed = 0;
  int        src_off    = 0;  // compensate for saved (optimized away) ptr advancement.
  int        dst_off    = 0;  // compensate for saved (optimized away) ptr advancement.
  bool       restore_inputs = false;
  bool       workreg_clear  = false;

  if ((len >= 32) && VM_Version::has_VectorFacility()) {
    const int  min_vcnt     = 32;          // Minimum #characters required to use vector instructions.
                                           // Otherwise just do nothing in vector mode.
                                           // Must be multiple of vector register length (16 bytes = 128 bits).
    const int  log_min_vcnt = exact_log2(min_vcnt);
    const int  iterations   = (len - nprocessed) >> log_min_vcnt;
    nprocessed             += iterations << log_min_vcnt;
    Label      VectorLoop;

    if (iterations == 1) {
      z_vlm(Z_V20, Z_V21, 0+src_off, Rsrc);  // get next 32 characters (single-byte)
      z_vuplhb(Z_V22, Z_V20);                // V2 <- (expand) V0(high)
      z_vupllb(Z_V23, Z_V20);                // V3 <- (expand) V0(low)
      z_vuplhb(Z_V24, Z_V21);                // V4 <- (expand) V1(high)
      z_vupllb(Z_V25, Z_V21);                // V5 <- (expand) V1(low)
      z_vstm(Z_V22, Z_V25, 0+dst_off, Rdst); // store next 32 bytes

      src_off += min_vcnt;
      dst_off += min_vcnt*2;
    } else {
      restore_inputs = true;

      z_lgfi(Rix, len>>log_min_vcnt);
      bind(VectorLoop);
        z_vlm(Z_V20, Z_V21, 0, Rsrc);        // get next 32 characters (single-byte)
        add2reg(Rsrc, min_vcnt);

        z_vuplhb(Z_V22, Z_V20);              // V2 <- (expand) V0(high)
        z_vupllb(Z_V23, Z_V20);              // V3 <- (expand) V0(low)
        z_vuplhb(Z_V24, Z_V21);              // V4 <- (expand) V1(high)
        z_vupllb(Z_V25, Z_V21);              // V5 <- (expand) V1(low)
        z_vstm(Z_V22, Z_V25, 0, Rdst);       // store next 32 bytes
        add2reg(Rdst, min_vcnt*2);

        z_brct(Rix, VectorLoop);
    }
  }

  if (((len-nprocessed) >= 16) && VM_Version::has_VectorFacility()) {
    const int  min_vcnt     = 16;          // Minimum #characters required to use vector instructions.
                                           // Otherwise just do nothing in vector mode.
                                           // Must be multiple of vector register length (16 bytes = 128 bits).
    const int  log_min_vcnt = exact_log2(min_vcnt);
    const int  iterations   = (len - nprocessed) >> log_min_vcnt;
    nprocessed             += iterations << log_min_vcnt;
    assert(iterations == 1, "must be!");

    z_vl(Z_V20, 0+src_off, Z_R0, Rsrc);    // get next 16 characters (single-byte)
    z_vuplhb(Z_V22, Z_V20);                // V2 <- (expand) V0(high)
    z_vupllb(Z_V23, Z_V20);                // V3 <- (expand) V0(low)
    z_vstm(Z_V22, Z_V23, 0+dst_off, Rdst); // store next 32 bytes

    src_off += min_vcnt;
    dst_off += min_vcnt*2;
  }

  if ((len-nprocessed) > 8) {
    const int  min_cnt     =  8;           // Minimum #characters required to use unrolled scalar loop.
                                           // Otherwise just do nothing in unrolled scalar mode.
                                           // Must be multiple of 8.
    const int  log_min_cnt = exact_log2(min_cnt);
    const int  iterations  = (len - nprocessed) >> log_min_cnt;
    nprocessed     += iterations << log_min_cnt;

    //---<  avoid loop overhead/ptr increment for small # iterations  >---
    if (iterations <= 2) {
      clear_reg(Z_R0);
      clear_reg(Z_R1);
      workreg_clear = true;

      z_icmh(Z_R0, 5, 0+src_off, Rsrc);
      z_icmh(Z_R1, 5, 4+src_off, Rsrc);
      z_icm(Z_R0,  5, 2+src_off, Rsrc);
      z_icm(Z_R1,  5, 6+src_off, Rsrc);
      z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);

      src_off += min_cnt;
      dst_off += min_cnt*2;
    }

    if (iterations == 2) {
      z_icmh(Z_R0, 5, 0+src_off, Rsrc);
      z_icmh(Z_R1, 5, 4+src_off, Rsrc);
      z_icm(Z_R0,  5, 2+src_off, Rsrc);
      z_icm(Z_R1,  5, 6+src_off, Rsrc);
      z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);

      src_off += min_cnt;
      dst_off += min_cnt*2;
    }

    if (iterations > 2) {
      Label      UnrolledLoop;
      restore_inputs  = true;

      clear_reg(Z_R0);
      clear_reg(Z_R1);
      workreg_clear = true;

      z_lgfi(Rix, iterations);
      bind(UnrolledLoop);
        z_icmh(Z_R0, 5, 0, Rsrc);
        z_icmh(Z_R1, 5, 4, Rsrc);
        z_icm(Z_R0,  5, 2, Rsrc);
        z_icm(Z_R1,  5, 6, Rsrc);
        add2reg(Rsrc, min_cnt);

        z_stmg(Z_R0, Z_R1, 0, Rdst);
        add2reg(Rdst, min_cnt*2);

        z_brct(Rix, UnrolledLoop);
    }
  }

  if ((len-nprocessed) > 0) {
    switch (len-nprocessed) {
      case 8:
        if (!workreg_clear) {
          clear_reg(Z_R0);
          clear_reg(Z_R1);
        }
        z_icmh(Z_R0, 5, 0+src_off, Rsrc);
        z_icmh(Z_R1, 5, 4+src_off, Rsrc);
        z_icm(Z_R0,  5, 2+src_off, Rsrc);
        z_icm(Z_R1,  5, 6+src_off, Rsrc);
        z_stmg(Z_R0, Z_R1, 0+dst_off, Rdst);
        break;
      case 7:
        if (!workreg_clear) {
          clear_reg(Z_R0);
          clear_reg(Z_R1);
        }
        clear_reg(Rix);
        z_icm(Z_R0,  5, 0+src_off, Rsrc);
        z_icm(Z_R1,  5, 2+src_off, Rsrc);
        z_icm(Rix,   5, 4+src_off, Rsrc);
        z_stm(Z_R0,  Z_R1, 0+dst_off, Rdst);
        z_llc(Z_R0,  6+src_off, Z_R0, Rsrc);
        z_st(Rix,    8+dst_off, Z_R0, Rdst);
        z_sth(Z_R0, 12+dst_off, Z_R0, Rdst);
        break;
      case 6:
        if (!workreg_clear) {
          clear_reg(Z_R0);
          clear_reg(Z_R1);
        }
        clear_reg(Rix);
        z_icm(Z_R0, 5, 0+src_off, Rsrc);
        z_icm(Z_R1, 5, 2+src_off, Rsrc);
        z_icm(Rix,  5, 4+src_off, Rsrc);
        z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
        z_st(Rix,   8+dst_off, Z_R0, Rdst);
        break;
      case 5:
        if (!workreg_clear) {
          clear_reg(Z_R0);
          clear_reg(Z_R1);
        }
        z_icm(Z_R0, 5, 0+src_off, Rsrc);
        z_icm(Z_R1, 5, 2+src_off, Rsrc);
        z_llc(Rix,  4+src_off, Z_R0, Rsrc);
        z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
        z_sth(Rix,  8+dst_off, Z_R0, Rdst);
        break;
      case 4:
        if (!workreg_clear) {
          clear_reg(Z_R0);
          clear_reg(Z_R1);
        }
        z_icm(Z_R0, 5, 0+src_off, Rsrc);
        z_icm(Z_R1, 5, 2+src_off, Rsrc);
        z_stm(Z_R0, Z_R1, 0+dst_off, Rdst);
        break;
      case 3:
        if (!workreg_clear) {
          clear_reg(Z_R0);
        }
        z_llc(Z_R1, 2+src_off, Z_R0, Rsrc);
        z_icm(Z_R0, 5, 0+src_off, Rsrc);
        z_sth(Z_R1, 4+dst_off, Z_R0, Rdst);
        z_st(Z_R0,  0+dst_off, Rdst);
        break;
      case 2:
        z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
        z_llc(Z_R1, 1+src_off, Z_R0, Rsrc);
        z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
        z_sth(Z_R1, 2+dst_off, Z_R0, Rdst);
        break;
      case 1:
        z_llc(Z_R0, 0+src_off, Z_R0, Rsrc);
        z_sth(Z_R0, 0+dst_off, Z_R0, Rdst);
        break;
      default:
        guarantee(false, "Impossible");
        break;
    }
    src_off   +=  len-nprocessed;
    dst_off   += (len-nprocessed)*2;
    nprocessed = len;
  }

  //---< restore modified input registers  >---
  if ((nprocessed > 0) && restore_inputs) {
    z_agfi(Rsrc, -(nprocessed-src_off));
    if (nprocessed < 1000000000) { // avoid int overflow
      z_agfi(Rdst, -(nprocessed*2-dst_off));
    } else {
      z_agfi(Rdst, -(nprocessed-dst_off));
      z_agfi(Rdst, -nprocessed);
    }
  }

  BLOCK_COMMENT("} string_inflate_const");
  return offset() - block_start;
}

// Returns the number of non-negative bytes (aka US-ASCII characters) found
// before the first negative byte is encountered.
unsigned int C2_MacroAssembler::count_positives(Register result, Register src, Register cnt, Register tmp) {
  const unsigned int block_start   = offset();
  const unsigned int byte_mask     = 0x80;
  const unsigned int twobyte_mask  = byte_mask<<8 | byte_mask;
  const unsigned int unroll_factor = 16;
  const unsigned int log_unroll_factor = exact_log2(unroll_factor);
  Register pos  = src;     // current position in src array, restored at end
  Register ctr  = result;  // loop counter, result value
  Register mask = tmp;     // holds the sign detection mask
  Label unrolledLoop, unrolledDone, byteLoop, allDone;

  assert_different_registers(result, src, cnt, tmp);

  BLOCK_COMMENT("count_positives {");

  lgr_if_needed(pos, src);              // current position in src array
  z_srak(ctr, cnt, log_unroll_factor);  // # iterations of unrolled loop
  z_brnh(unrolledDone);                 // array too short for unrolled loop

  z_iilf(mask, twobyte_mask<<16 | twobyte_mask);
  z_iihf(mask, twobyte_mask<<16 | twobyte_mask);

  bind(unrolledLoop);
    z_lmg(Z_R0, Z_R1, 0, pos);
    z_ogr(Z_R0, Z_R1);
    z_ngr(Z_R0, mask);
    z_brne(unrolledDone);               // There is a negative byte somewhere.
                                        // ctr and pos are not updated yet ->
                                        // delegate finding correct pos to byteLoop.
    add2reg(pos, unroll_factor);
    z_brct(ctr, unrolledLoop);

  // Once we arrive here, we have to examine at most (unroll_factor - 1) bytes more.
  // We then either have reached the end of the array or we hit a negative byte.
  bind(unrolledDone);
  z_sll(ctr, log_unroll_factor);        // calculate # bytes not processed by unrolled loop
                                        // > 0 only if a negative byte was found
  z_lr(Z_R0, cnt);                      // calculate remainder bytes
  z_nilf(Z_R0, unroll_factor - 1);
  z_ar(ctr, Z_R0);                      // remaining bytes
  z_brnh(allDone);                      // shortcut if nothing left to do

  bind(byteLoop);
    z_cli(0, pos, byte_mask);           // unsigned comparison! byte@pos must be smaller that byte_mask
    z_brnl(allDone);                    // negative byte found.

    add2reg(pos, 1);
    z_brct(ctr, byteLoop);

  bind(allDone);

  z_srk(ctr, cnt, ctr);                 // # bytes actually processed (= cnt or index of first negative byte)
  z_sgfr(pos, ctr);                     // restore src
  z_lgfr(result, ctr);                  // unnecessary. Only there to be sure the high word has a defined state.

  BLOCK_COMMENT("} count_positives");

  return offset() - block_start;
}

// kill: cnt1, cnt2, odd_reg, even_reg; early clobber: result
unsigned int C2_MacroAssembler::string_compare(Register str1, Register str2,
                                               Register cnt1, Register cnt2,
                                               Register odd_reg, Register even_reg, Register result, int ae) {
  int block_start = offset();

  assert_different_registers(str1, cnt1, cnt2, odd_reg, even_reg, result);
  assert_different_registers(str2, cnt1, cnt2, odd_reg, even_reg, result);

  // If strings are equal up to min length, return the length difference.
  const Register diff = result, // Pre-set result with length difference.
                 min  = cnt1,   // min number of bytes
                 tmp  = cnt2;

  // Note: Making use of the fact that compareTo(a, b) == -compareTo(b, a)
  // we interchange str1 and str2 in the UL case and negate the result.
  // Like this, str1 is always latin1 encoded, except for the UU case.
  // In addition, we need 0 (or sign which is 0) extend when using 64 bit register.
  const bool used_as_LU = (ae == StrIntrinsicNode::LU || ae == StrIntrinsicNode::UL);

  BLOCK_COMMENT("string_compare {");

  if (used_as_LU) {
    z_srl(cnt2, 1);
  }

  // See if the lengths are different, and calculate min in cnt1.
  // Save diff in case we need it for a tie-breaker.

  // diff = cnt1 - cnt2
  if (VM_Version::has_DistinctOpnds()) {
    z_srk(diff, cnt1, cnt2);
  } else {
    z_lr(diff, cnt1);
    z_sr(diff, cnt2);
  }
  if (str1 != str2) {
    if (VM_Version::has_LoadStoreConditional()) {
      z_locr(min, cnt2, Assembler::bcondHigh);
    } else {
      Label Lskip;
      z_brl(Lskip);    // min ok if cnt1 < cnt2
      z_lr(min, cnt2); // min = cnt2
      bind(Lskip);
    }
  }

  if (ae == StrIntrinsicNode::UU) {
    z_sra(diff, 1);
  }
  if (str1 != str2) {
    Label Ldone;
    if (used_as_LU) {
      // Loop which searches the first difference character by character.
      Label Lloop;
      const Register ind1 = Z_R1,
                     ind2 = min;
      int stride1 = 1, stride2 = 2; // See comment above.

      // ind1: index, even_reg: index increment, odd_reg: index limit
      z_llilf(ind1, (unsigned int)(-stride1));
      z_lhi(even_reg, stride1);
      add2reg(odd_reg, -stride1, min);
      clear_reg(ind2); // kills min

      bind(Lloop);
      z_brxh(ind1, even_reg, Ldone);
      z_llc(tmp, Address(str1, ind1));
      z_llh(Z_R0, Address(str2, ind2));
      z_ahi(ind2, stride2);
      z_sr(tmp, Z_R0);
      z_bre(Lloop);

      z_lr(result, tmp);

    } else {
      // Use clcle in fast loop (only for same encoding).
      z_lgr(Z_R0, str1);
      z_lgr(even_reg, str2);
      z_llgfr(Z_R1, min);
      z_llgfr(odd_reg, min);

      if (ae == StrIntrinsicNode::LL) {
        compare_long_ext(Z_R0, even_reg, 0);
      } else {
        compare_long_uni(Z_R0, even_reg, 0);
      }
      z_bre(Ldone);
      z_lgr(Z_R1, Z_R0);
      if (ae == StrIntrinsicNode::LL) {
        z_llc(Z_R0, Address(even_reg));
        z_llc(result, Address(Z_R1));
      } else {
        z_llh(Z_R0, Address(even_reg));
        z_llh(result, Address(Z_R1));
      }
      z_sr(result, Z_R0);
    }

    // Otherwise, return the difference between the first mismatched chars.
    bind(Ldone);
  }

  if (ae == StrIntrinsicNode::UL) {
    z_lcr(result, result); // Negate result (see note above).
  }

  BLOCK_COMMENT("} string_compare");

  return offset() - block_start;
}

unsigned int C2_MacroAssembler::array_equals(bool is_array_equ, Register ary1, Register ary2, Register limit,
                                             Register odd_reg, Register even_reg, Register result, bool is_byte) {
  int block_start = offset();

  BLOCK_COMMENT("array_equals {");

  assert_different_registers(ary1, limit, odd_reg, even_reg);
  assert_different_registers(ary2, limit, odd_reg, even_reg);

  Label Ldone, Ldone_true, Ldone_false, Lclcle, CLC_template;
  int base_offset = 0;

  if (ary1 != ary2) {
    if (is_array_equ) {
      base_offset = arrayOopDesc::base_offset_in_bytes(is_byte ? T_BYTE : T_CHAR);

      // Return true if the same array.
      compareU64_and_branch(ary1, ary2, Assembler::bcondEqual, Ldone_true);

      // Return false if one of them is NULL.
      compareU64_and_branch(ary1, (intptr_t)0, Assembler::bcondEqual, Ldone_false);
      compareU64_and_branch(ary2, (intptr_t)0, Assembler::bcondEqual, Ldone_false);

      // Load the lengths of arrays.
      z_llgf(odd_reg, Address(ary1, arrayOopDesc::length_offset_in_bytes()));

      // Return false if the two arrays are not equal length.
      z_c(odd_reg, Address(ary2, arrayOopDesc::length_offset_in_bytes()));
      z_brne(Ldone_false);

      // string len in bytes (right operand)
      if (!is_byte) {
        z_chi(odd_reg, 128);
        z_sll(odd_reg, 1); // preserves flags
        z_brh(Lclcle);
      } else {
        compareU32_and_branch(odd_reg, (intptr_t)256, Assembler::bcondHigh, Lclcle);
      }
    } else {
      z_llgfr(odd_reg, limit); // Need to zero-extend prior to using the value.
      compareU32_and_branch(limit, (intptr_t)256, Assembler::bcondHigh, Lclcle);
    }

    // Use clc instruction for up to 256 bytes.
    {
      Register str1_reg = ary1,
          str2_reg = ary2;
      if (is_array_equ) {
        str1_reg = Z_R1;
        str2_reg = even_reg;
        add2reg(str1_reg, base_offset, ary1); // string addr (left operand)
        add2reg(str2_reg, base_offset, ary2); // string addr (right operand)
      }
      z_ahi(odd_reg, -1); // Clc uses decremented limit. Also compare result to 0.
      z_brl(Ldone_true);
      // Note: We could jump to the template if equal.

      assert(VM_Version::has_ExecuteExtensions(), "unsupported hardware");
      z_exrl(odd_reg, CLC_template);
      z_bre(Ldone_true);
      // fall through

      bind(Ldone_false);
      clear_reg(result);
      z_bru(Ldone);

      bind(CLC_template);
      z_clc(0, 0, str1_reg, 0, str2_reg);
    }

    // Use clcle instruction.
    {
      bind(Lclcle);
      add2reg(even_reg, base_offset, ary2); // string addr (right operand)
      add2reg(Z_R0, base_offset, ary1);     // string addr (left operand)

      z_lgr(Z_R1, odd_reg); // string len in bytes (left operand)
      if (is_byte) {
        compare_long_ext(Z_R0, even_reg, 0);
      } else {
        compare_long_uni(Z_R0, even_reg, 0);
      }
      z_lghi(result, 0); // Preserve flags.
      z_brne(Ldone);
    }
  }
  // fall through

  bind(Ldone_true);
  z_lghi(result, 1); // All characters are equal.
  bind(Ldone);

  BLOCK_COMMENT("} array_equals");

  return offset() - block_start;
}

// kill: haycnt, needlecnt, odd_reg, even_reg; early clobber: result
unsigned int C2_MacroAssembler::string_indexof(Register result, Register haystack, Register haycnt,
                                               Register needle, Register needlecnt, int needlecntval,
                                               Register odd_reg, Register even_reg, int ae) {
  int block_start = offset();

  // Ensure 0<needlecnt<=haycnt in ideal graph as prerequisite!
  assert(ae != StrIntrinsicNode::LU, "Invalid encoding");
  const int h_csize = (ae == StrIntrinsicNode::LL) ? 1 : 2;
  const int n_csize = (ae == StrIntrinsicNode::UU) ? 2 : 1;
  Label L_needle1, L_Found, L_NotFound;

  BLOCK_COMMENT("string_indexof {");

  if (needle == haystack) {
    z_lhi(result, 0);
  } else {

  // Load first character of needle (R0 used by search_string instructions).
  if (n_csize == 2) { z_llgh(Z_R0, Address(needle)); } else { z_llgc(Z_R0, Address(needle)); }

  // Compute last haystack addr to use if no match gets found.
  if (needlecnt != noreg) { // variable needlecnt
    z_ahi(needlecnt, -1); // Remaining characters after first one.
    z_sr(haycnt, needlecnt); // Compute index succeeding last element to compare.
    if (n_csize == 2) { z_sll(needlecnt, 1); } // In bytes.
  } else { // constant needlecnt
    assert((needlecntval & 0x7fff) == needlecntval, "must be positive simm16 immediate");
    // Compute index succeeding last element to compare.
    if (needlecntval != 1) { z_ahi(haycnt, 1 - needlecntval); }
  }

  z_llgfr(haycnt, haycnt); // Clear high half.
  z_lgr(result, haystack); // Final result will be computed from needle start pointer.
  if (h_csize == 2) { z_sll(haycnt, 1); } // Scale to number of bytes.
  z_agr(haycnt, haystack); // Point to address succeeding last element (haystack+scale*(haycnt-needlecnt+1)).

  if (h_csize != n_csize) {
    assert(ae == StrIntrinsicNode::UL, "Invalid encoding");

    if (needlecnt != noreg || needlecntval != 1) {
      if (needlecnt != noreg) {
        compare32_and_branch(needlecnt, (intptr_t)0, Assembler::bcondEqual, L_needle1);
      }

      // Main Loop: UL version (now we have at least 2 characters).
      Label L_OuterLoop, L_InnerLoop, L_Skip;
      bind(L_OuterLoop); // Search for 1st 2 characters.
      z_lgr(Z_R1, haycnt);
      MacroAssembler::search_string_uni(Z_R1, result);
      z_brc(Assembler::bcondNotFound, L_NotFound);
      z_lgr(result, Z_R1);

      z_lghi(Z_R1, n_csize);
      z_lghi(even_reg, h_csize);
      bind(L_InnerLoop);
      z_llgc(odd_reg, Address(needle, Z_R1));
      z_ch(odd_reg, Address(result, even_reg));
      z_brne(L_Skip);
      if (needlecnt != noreg) { z_cr(Z_R1, needlecnt); } else { z_chi(Z_R1, needlecntval - 1); }
      z_brnl(L_Found);
      z_aghi(Z_R1, n_csize);
      z_aghi(even_reg, h_csize);
      z_bru(L_InnerLoop);

      bind(L_Skip);
      z_aghi(result, h_csize); // This is the new address we want to use for comparing.
      z_bru(L_OuterLoop);
    }

  } else {
    const intptr_t needle_bytes = (n_csize == 2) ? ((needlecntval - 1) << 1) : (needlecntval - 1);
    Label L_clcle;

    if (needlecnt != noreg || (needlecntval != 1 && needle_bytes <= 256)) {
      if (needlecnt != noreg) {
        compare32_and_branch(needlecnt, 256, Assembler::bcondHigh, L_clcle);
        z_ahi(needlecnt, -1); // remaining bytes -1 (for CLC)
        z_brl(L_needle1);
      }

      // Main Loop: clc version (now we have at least 2 characters).
      Label L_OuterLoop, CLC_template;
      bind(L_OuterLoop); // Search for 1st 2 characters.
      z_lgr(Z_R1, haycnt);
      if (h_csize == 1) {
        MacroAssembler::search_string(Z_R1, result);
      } else {
        MacroAssembler::search_string_uni(Z_R1, result);
      }
      z_brc(Assembler::bcondNotFound, L_NotFound);
      z_lgr(result, Z_R1);

      if (needlecnt != noreg) {
        assert(VM_Version::has_ExecuteExtensions(), "unsupported hardware");
        z_exrl(needlecnt, CLC_template);
      } else {
        z_clc(h_csize, needle_bytes -1, Z_R1, n_csize, needle);
      }
      z_bre(L_Found);
      z_aghi(result, h_csize); // This is the new address we want to use for comparing.
      z_bru(L_OuterLoop);

      if (needlecnt != noreg) {
        bind(CLC_template);
        z_clc(h_csize, 0, Z_R1, n_csize, needle);
      }
    }

    if (needlecnt != noreg || needle_bytes > 256) {
      bind(L_clcle);

      // Main Loop: clcle version (now we have at least 256 bytes).
      Label L_OuterLoop, CLC_template;
      bind(L_OuterLoop); // Search for 1st 2 characters.
      z_lgr(Z_R1, haycnt);
      if (h_csize == 1) {
        MacroAssembler::search_string(Z_R1, result);
      } else {
        MacroAssembler::search_string_uni(Z_R1, result);
      }
      z_brc(Assembler::bcondNotFound, L_NotFound);

      add2reg(Z_R0, n_csize, needle);
      add2reg(even_reg, h_csize, Z_R1);
      z_lgr(result, Z_R1);
      if (needlecnt != noreg) {
        z_llgfr(Z_R1, needlecnt); // needle len in bytes (left operand)
        z_llgfr(odd_reg, needlecnt);
      } else {
        load_const_optimized(Z_R1, needle_bytes);
        if (Immediate::is_simm16(needle_bytes)) { z_lghi(odd_reg, needle_bytes); } else { z_lgr(odd_reg, Z_R1); }
      }
      if (h_csize == 1) {
        compare_long_ext(Z_R0, even_reg, 0);
      } else {
        compare_long_uni(Z_R0, even_reg, 0);
      }
      z_bre(L_Found);

      if (n_csize == 2) { z_llgh(Z_R0, Address(needle)); } else { z_llgc(Z_R0, Address(needle)); } // Reload.
      z_aghi(result, h_csize); // This is the new address we want to use for comparing.
      z_bru(L_OuterLoop);
    }
  }

  if (needlecnt != noreg || needlecntval == 1) {
    bind(L_needle1);

    // Single needle character version.
    if (h_csize == 1) {
      MacroAssembler::search_string(haycnt, result);
    } else {
      MacroAssembler::search_string_uni(haycnt, result);
    }
    z_lgr(result, haycnt);
    z_brc(Assembler::bcondFound, L_Found);
  }

  bind(L_NotFound);
  add2reg(result, -1, haystack); // Return -1.

  bind(L_Found); // Return index (or -1 in fallthrough case).
  z_sgr(result, haystack);
  if (h_csize == 2) { z_srag(result, result, exact_log2(sizeof(jchar))); }
  }
  BLOCK_COMMENT("} string_indexof");

  return offset() - block_start;
}

// early clobber: result
unsigned int C2_MacroAssembler::string_indexof_char(Register result, Register haystack, Register haycnt,
                                                    Register needle, jchar needleChar, Register odd_reg, Register even_reg, bool is_byte) {
  int block_start = offset();

  BLOCK_COMMENT("string_indexof_char {");

  if (needle == haystack) {
    z_lhi(result, 0);
  } else {

  Label Ldone;

  z_llgfr(odd_reg, haycnt);  // Preset loop ctr/searchrange end.
  if (needle == noreg) {
    load_const_optimized(Z_R0, (unsigned long)needleChar);
  } else {
    if (is_byte) {
      z_llgcr(Z_R0, needle); // First (and only) needle char.
    } else {
      z_llghr(Z_R0, needle); // First (and only) needle char.
    }
  }

  if (!is_byte) {
    z_agr(odd_reg, odd_reg); // Calc #bytes to be processed with SRSTU.
  }

  z_lgr(even_reg, haystack); // haystack addr
  z_agr(odd_reg, haystack);  // First char after range end.
  z_lghi(result, -1);

  if (is_byte) {
    MacroAssembler::search_string(odd_reg, even_reg);
  } else {
    MacroAssembler::search_string_uni(odd_reg, even_reg);
  }
  z_brc(Assembler::bcondNotFound, Ldone);
  if (is_byte) {
    if (VM_Version::has_DistinctOpnds()) {
      z_sgrk(result, odd_reg, haystack);
    } else {
      z_sgr(odd_reg, haystack);
      z_lgr(result, odd_reg);
    }
  } else {
    z_slgr(odd_reg, haystack);
    z_srlg(result, odd_reg, exact_log2(sizeof(jchar)));
  }

  bind(Ldone);
  }
  BLOCK_COMMENT("} string_indexof_char");

  return offset() - block_start;
}

quality97%

¤ Dauer der Verarbeitung: 0.54 Sekunden (vorverarbeitet) ¤

Wurzel

Suchen

Beweissystem der NASA

Beweissystem Isabelle

NIST Cobol Testsuite

Cephes Mathematical Library

Wiener Entwicklungsmethode

Haftungshinweis

Die Informationen auf dieser Webseite wurden nach bestem Wissen sorgfältig zusammengestellt. Es wird jedoch weder Vollständigkeit, noch Richtigkeit, noch Qualität der bereit gestellten Informationen zugesichert.

Bemerkung:

Die farbliche Syntaxdarstellung ist noch experimentell.