/* Subroutines used to expand string and block move, clear,
compare and other operations for PowerPC.
Copyright (C) 1991-2022 Free Software Foundation, Inc.
This file is part of GCC.
GCC is free software; you can redistribute it and/or modify it
under the terms of the GNU General Public License as published
by the Free Software Foundation; either version 3, or (at your
option) any later version.
GCC is distributed in the hope that it will be useful, but WITHOUT
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public
License for more details.
You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3. If not see
. */
#define IN_TARGET_CODE 1
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "rtl.h"
#include "tree.h"
#include "memmodel.h"
#include "tm_p.h"
#include "ira.h"
#include "print-tree.h"
#include "varasm.h"
#include "explow.h"
#include "expr.h"
#include "output.h"
#include "target.h"
#include "profile-count.h"
#include "predict.h"
/* Expand a block clear operation, and return 1 if successful. Return 0
if we should let the compiler generate normal code.
operands[0] is the destination
operands[1] is the length
operands[3] is the alignment */
int
expand_block_clear (rtx operands[])
{
rtx orig_dest = operands[0];
rtx bytes_rtx = operands[1];
rtx align_rtx = operands[3];
bool constp = CONST_INT_P (bytes_rtx);
HOST_WIDE_INT align;
HOST_WIDE_INT bytes;
int offset;
int clear_bytes;
int clear_step;
/* If this is not a fixed size move, just call memcpy */
if (! constp)
return 0;
/* This must be a fixed size alignment */
gcc_assert (CONST_INT_P (align_rtx));
align = INTVAL (align_rtx) * BITS_PER_UNIT;
/* Anything to clear? */
bytes = INTVAL (bytes_rtx);
if (bytes <= 0)
return 1;
/* Use the builtin memset after a point, to avoid huge code bloat.
When optimize_size, avoid any significant code bloat; calling
memset is about 4 instructions, so allow for one instruction to
load zero and three to do clearing. */
if (TARGET_ALTIVEC && (align >= 128 || TARGET_EFFICIENT_UNALIGNED_VSX))
clear_step = 16;
else if (TARGET_POWERPC64 && (align >= 64 || !STRICT_ALIGNMENT))
clear_step = 8;
else
clear_step = 4;
if (optimize_size && bytes > 3 * clear_step)
return 0;
if (! optimize_size && bytes > 8 * clear_step)
return 0;
bool unaligned_vsx_ok = (bytes >= 32 && TARGET_EFFICIENT_UNALIGNED_VSX);
for (offset = 0; bytes > 0; offset += clear_bytes, bytes -= clear_bytes)
{
machine_mode mode = BLKmode;
rtx dest;
if (TARGET_ALTIVEC
&& (bytes >= 16 && (align >= 128 || unaligned_vsx_ok)))
{
clear_bytes = 16;
mode = V4SImode;
}
else if (bytes >= 8 && TARGET_POWERPC64
&& (align >= 64 || !STRICT_ALIGNMENT))
{
clear_bytes = 8;
mode = DImode;
if (offset == 0 && align < 64)
{
rtx addr;
/* If the address form is reg+offset with offset not a
multiple of four, reload into reg indirect form here
rather than waiting for reload. This way we get one
reload, not one per store. */
addr = XEXP (orig_dest, 0);
if ((GET_CODE (addr) == PLUS || GET_CODE (addr) == LO_SUM)
&& CONST_INT_P (XEXP (addr, 1))
&& (INTVAL (XEXP (addr, 1)) & 3) != 0)
{
addr = copy_addr_to_reg (addr);
orig_dest = replace_equiv_address (orig_dest, addr);
}
}
}
else if (bytes >= 4 && (align >= 32 || !STRICT_ALIGNMENT))
{ /* move 4 bytes */
clear_bytes = 4;
mode = SImode;
}
else if (bytes >= 2 && (align >= 16 || !STRICT_ALIGNMENT))
{ /* move 2 bytes */
clear_bytes = 2;
mode = HImode;
}
else /* move 1 byte at a time */
{
clear_bytes = 1;
mode = QImode;
}
dest = adjust_address (orig_dest, mode, offset);
emit_move_insn (dest, CONST0_RTX (mode));
}
return 1;
}
/* Figure out the correct instructions to generate to load data for
block compare. MODE is used for the read from memory, and
data is zero extended if REG is wider than MODE. If LE code
is being generated, bswap loads are used.
REG is the destination register to move the data into.
MEM is the memory block being read.
MODE is the mode of memory to use for the read. */
static void
do_load_for_compare (rtx reg, rtx mem, machine_mode mode)
{
switch (GET_MODE (reg))
{
case E_V16QImode:
switch (mode)
{
case E_V16QImode:
if (!BYTES_BIG_ENDIAN)
{
if (TARGET_P9_VECTOR)
emit_insn (gen_vsx_ld_elemrev_v16qi_internal (reg, mem));
else
{
rtx reg_v2di = simplify_gen_subreg (V2DImode, reg,
V16QImode, 0);
gcc_assert (MEM_P (mem));
rtx addr = XEXP (mem, 0);
rtx mem_v2di = gen_rtx_MEM (V2DImode, addr);
MEM_COPY_ATTRIBUTES (mem_v2di, mem);
set_mem_size (mem, GET_MODE_SIZE (V2DImode));
emit_insn (gen_vsx_ld_elemrev_v2di (reg_v2di, mem_v2di));
}
}
else
emit_insn (gen_vsx_movv2di_64bit (reg, mem));
break;
default:
gcc_unreachable ();
}
break;
case E_DImode:
switch (mode)
{
case E_QImode:
emit_insn (gen_zero_extendqidi2 (reg, mem));
break;
case E_HImode:
{
rtx src = mem;
if (!BYTES_BIG_ENDIAN)
{
src = gen_reg_rtx (HImode);
emit_insn (gen_bswaphi2 (src, mem));
}
emit_insn (gen_zero_extendhidi2 (reg, src));
break;
}
case E_SImode:
{
rtx src = mem;
if (!BYTES_BIG_ENDIAN)
{
src = gen_reg_rtx (SImode);
emit_insn (gen_bswapsi2 (src, mem));
}
emit_insn (gen_zero_extendsidi2 (reg, src));
}
break;
case E_DImode:
if (!BYTES_BIG_ENDIAN)
emit_insn (gen_bswapdi2 (reg, mem));
else
emit_insn (gen_movdi (reg, mem));
break;
default:
gcc_unreachable ();
}
break;
case E_SImode:
switch (mode)
{
case E_QImode:
emit_insn (gen_zero_extendqisi2 (reg, mem));
break;
case E_HImode:
{
rtx src = mem;
if (!BYTES_BIG_ENDIAN)
{
src = gen_reg_rtx (HImode);
emit_insn (gen_bswaphi2 (src, mem));
}
emit_insn (gen_zero_extendhisi2 (reg, src));
break;
}
case E_SImode:
if (!BYTES_BIG_ENDIAN)
emit_insn (gen_bswapsi2 (reg, mem));
else
emit_insn (gen_movsi (reg, mem));
break;
case E_DImode:
/* DImode is larger than the destination reg so is not expected. */
gcc_unreachable ();
break;
default:
gcc_unreachable ();
}
break;
case E_QImode:
gcc_assert (mode == E_QImode);
emit_move_insn (reg, mem);
break;
default:
gcc_unreachable ();
break;
}
}
/* Select the mode to be used for reading the next chunk of bytes
in the compare.
OFFSET is the current read offset from the beginning of the block.
BYTES is the number of bytes remaining to be read.
ALIGN is the minimum alignment of the memory blocks being compared in bytes. */
static machine_mode
select_block_compare_mode (unsigned HOST_WIDE_INT offset,
unsigned HOST_WIDE_INT bytes,
unsigned HOST_WIDE_INT align)
{
/* First see if we can do a whole load unit
as that will be more efficient than a larger load + shift. */
/* If big, use biggest chunk.
If exactly chunk size, use that size.
If remainder can be done in one piece with shifting, do that.
Do largest chunk possible without violating alignment rules. */
/* The most we can read without potential page crossing. */
unsigned HOST_WIDE_INT maxread = ROUND_UP (bytes, align);
/* If we have an LE target without ldbrx and word_mode is DImode,
then we must avoid using word_mode. */
int word_mode_ok = !(!BYTES_BIG_ENDIAN && !TARGET_LDBRX
&& word_mode == DImode);
if (word_mode_ok && bytes >= UNITS_PER_WORD)
return word_mode;
else if (bytes == GET_MODE_SIZE (SImode))
return SImode;
else if (bytes == GET_MODE_SIZE (HImode))
return HImode;
else if (bytes == GET_MODE_SIZE (QImode))
return QImode;
else if (bytes < GET_MODE_SIZE (SImode)
&& TARGET_EFFICIENT_OVERLAPPING_UNALIGNED
&& offset >= GET_MODE_SIZE (SImode) - bytes)
/* This matches the case were we have SImode and 3 bytes
and offset >= 1 and permits us to move back one and overlap
with the previous read, thus avoiding having to shift
unwanted bytes off of the input. */
return SImode;
else if (word_mode_ok && bytes < UNITS_PER_WORD
&& TARGET_EFFICIENT_OVERLAPPING_UNALIGNED
&& offset >= UNITS_PER_WORD-bytes)
/* Similarly, if we can use DImode it will get matched here and
can do an overlapping read that ends at the end of the block. */
return word_mode;
else if (word_mode_ok && maxread >= UNITS_PER_WORD)
/* It is safe to do all remaining in one load of largest size,
possibly with a shift to get rid of unwanted bytes. */
return word_mode;
else if (maxread >= GET_MODE_SIZE (SImode))
/* It is safe to do all remaining in one SImode load,
possibly with a shift to get rid of unwanted bytes. */
return SImode;
else if (bytes > GET_MODE_SIZE (SImode))
return SImode;
else if (bytes > GET_MODE_SIZE (HImode))
return HImode;
/* final fallback is do one byte */
return QImode;
}
/* Compute the alignment of pointer+OFFSET where the original alignment
of pointer was BASE_ALIGN. */
static unsigned HOST_WIDE_INT
compute_current_alignment (unsigned HOST_WIDE_INT base_align,
unsigned HOST_WIDE_INT offset)
{
if (offset == 0)
return base_align;
return MIN (base_align, offset & -offset);
}
/* Prepare address and then do a load.
MODE is the mode to use for the load.
DEST is the destination register for the data.
ADDR is the address to be loaded.
ORIG_ADDR is the original address expression. */
static void
do_load_for_compare_from_addr (machine_mode mode, rtx dest, rtx addr,
rtx orig_addr)
{
rtx mem = gen_rtx_MEM (mode, addr);
MEM_COPY_ATTRIBUTES (mem, orig_addr);
set_mem_size (mem, GET_MODE_SIZE (mode));
do_load_for_compare (dest, mem, mode);
return;
}
/* Do a branch for an if/else decision.
CMPMODE is the mode to use for the comparison.
COMPARISON is the rtx code for the compare needed.
A is the first thing to be compared.
B is the second thing to be compared.
CR is the condition code reg input, or NULL_RTX.
TRUE_LABEL is the label to branch to if the condition is true.
P is the estimated branch probability for the branch.
The return value is the CR used for the comparison.
If CR is null_rtx, then a new register of CMPMODE is generated.
If A and B are both null_rtx, then CR must not be null, and the
compare is not generated so you can use this with a dot form insn. */
static void
do_ifelse (machine_mode cmpmode, rtx_code comparison,
rtx a, rtx b, rtx cr, rtx true_label, profile_probability br_prob)
{
gcc_assert ((a == NULL_RTX && b == NULL_RTX && cr != NULL_RTX)
|| (a != NULL_RTX && b != NULL_RTX));
if (cr != NULL_RTX)
gcc_assert (GET_MODE (cr) == cmpmode);
else
cr = gen_reg_rtx (cmpmode);
rtx label_ref = gen_rtx_LABEL_REF (VOIDmode, true_label);
if (a != NULL_RTX)
emit_move_insn (cr, gen_rtx_COMPARE (cmpmode, a, b));
rtx cmp_rtx = gen_rtx_fmt_ee (comparison, VOIDmode, cr, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx, label_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, br_prob);
JUMP_LABEL (j) = true_label;
LABEL_NUSES (true_label) += 1;
}
/* Emit an isel of the proper mode for DEST.
DEST is the isel destination register.
SRC1 is the isel source if CR is true.
SRC2 is the isel source if CR is false.
CR is the condition for the isel. */
static void
do_isel (rtx dest, rtx cmp, rtx src_t, rtx src_f, rtx cr)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_isel_signed_di (dest, cmp, src_t, src_f, cr));
else
emit_insn (gen_isel_signed_si (dest, cmp, src_t, src_f, cr));
}
/* Emit a subtract of the proper mode for DEST.
DEST is the destination register for the subtract.
SRC1 is the first subtract input.
SRC2 is the second subtract input.
Computes DEST = SRC1-SRC2. */
static void
do_sub3 (rtx dest, rtx src1, rtx src2)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_subdi3 (dest, src1, src2));
else
emit_insn (gen_subsi3 (dest, src1, src2));
}
/* Emit an add of the proper mode for DEST.
DEST is the destination register for the add.
SRC1 is the first add input.
SRC2 is the second add input.
Computes DEST = SRC1+SRC2. */
static void
do_add3 (rtx dest, rtx src1, rtx src2)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_adddi3 (dest, src1, src2));
else
emit_insn (gen_addsi3 (dest, src1, src2));
}
/* Emit an and of the proper mode for DEST.
DEST is the destination register for the and.
SRC1 is the first and input.
SRC2 is the second and input.
Computes DEST = SRC1&SRC2. */
static void
do_and3 (rtx dest, rtx src1, rtx src2)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_anddi3 (dest, src1, src2));
else
emit_insn (gen_andsi3 (dest, src1, src2));
}
/* Emit an cmpb of the proper mode for DEST.
DEST is the destination register for the cmpb.
SRC1 is the first input.
SRC2 is the second input.
Computes cmpb of SRC1, SRC2. */
static void
do_cmpb3 (rtx dest, rtx src1, rtx src2)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_cmpbdi3 (dest, src1, src2));
else
emit_insn (gen_cmpbsi3 (dest, src1, src2));
}
/* Emit a rotl of the proper mode for DEST.
DEST is the destination register for the and.
SRC1 is the first and input.
SRC2 is the second and input.
Computes DEST = SRC1 rotated left by SRC2. */
static void
do_rotl3 (rtx dest, rtx src1, rtx src2)
{
if (GET_MODE (dest) == DImode)
emit_insn (gen_rotldi3 (dest, src1, src2));
else
emit_insn (gen_rotlsi3 (dest, src1, src2));
}
/* Generate rtl for a load, shift, and compare of less than a full word.
LOAD_MODE is the machine mode for the loads.
DIFF is the reg for the difference.
CMP_REM is the reg containing the remaining bytes to compare.
DCOND is the CCUNS reg for the compare if we are doing P9 code with setb.
SRC1_ADDR is the first source address.
SRC2_ADDR is the second source address.
ORIG_SRC1 is the original first source block's address rtx.
ORIG_SRC2 is the original second source block's address rtx. */
static void
do_load_mask_compare (const machine_mode load_mode, rtx diff, rtx cmp_rem, rtx dcond,
rtx src1_addr, rtx src2_addr, rtx orig_src1, rtx orig_src2)
{
HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
rtx shift_amount = gen_reg_rtx (word_mode);
rtx d1 = gen_reg_rtx (word_mode);
rtx d2 = gen_reg_rtx (word_mode);
do_load_for_compare_from_addr (load_mode, d1, src1_addr, orig_src1);
do_load_for_compare_from_addr (load_mode, d2, src2_addr, orig_src2);
do_sub3 (shift_amount, GEN_INT (load_mode_size), cmp_rem);
if (word_mode == DImode)
{
emit_insn (gen_ashldi3 (shift_amount, shift_amount,
GEN_INT (LOG2_BITS_PER_UNIT)));
emit_insn (gen_lshrdi3 (d1, d1,
gen_lowpart (SImode, shift_amount)));
emit_insn (gen_lshrdi3 (d2, d2,
gen_lowpart (SImode, shift_amount)));
}
else
{
emit_insn (gen_ashlsi3 (shift_amount, shift_amount,
GEN_INT (LOG2_BITS_PER_UNIT)));
emit_insn (gen_lshrsi3 (d1, d1, shift_amount));
emit_insn (gen_lshrsi3 (d2, d2, shift_amount));
}
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1, d2);
emit_insn (gen_rtx_SET (dcond, cmp));
}
else
{
if (word_mode == DImode)
emit_insn (gen_subfdi3_carry (diff, d2, d1));
else
emit_insn (gen_subfsi3_carry (diff, d2, d1));
}
}
/* Generate rtl for an overlapping load and compare of less than a
full load_mode. This assumes that the previous word is part of the
block being compared so it's ok to back up part of a word so we can
compare the last unaligned full word that ends at the end of the block.
LOAD_MODE is the machine mode for the loads.
ISCONST tells whether the remaining length is a constant or in a register.
BYTES_REM is the remaining length if ISCONST is true.
DIFF is the reg for the difference.
CMP_REM is the reg containing the remaining bytes to compare if !ISCONST.
DCOND is the CCUNS reg for the compare if we are doing P9 code with setb.
SRC1_ADDR is the first source address.
SRC2_ADDR is the second source address.
ORIG_SRC1 is the original first source block's address rtx.
ORIG_SRC2 is the original second source block's address rtx. */
static void
do_overlap_load_compare (machine_mode load_mode, bool isConst,
HOST_WIDE_INT bytes_rem, rtx diff,
rtx cmp_rem, rtx dcond, rtx src1_addr, rtx src2_addr,
rtx orig_src1, rtx orig_src2)
{
HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
HOST_WIDE_INT addr_adj = load_mode_size - bytes_rem;
rtx d1 = gen_reg_rtx (word_mode);
rtx d2 = gen_reg_rtx (word_mode);
rtx addr1, addr2;
if (!isConst || addr_adj)
{
rtx adj_reg = gen_reg_rtx (word_mode);
if (isConst)
emit_move_insn (adj_reg, GEN_INT (-addr_adj));
else
{
rtx reg_lms = gen_reg_rtx (word_mode);
emit_move_insn (reg_lms, GEN_INT (load_mode_size));
do_sub3 (adj_reg, cmp_rem, reg_lms);
}
addr1 = gen_rtx_PLUS (word_mode, src1_addr, adj_reg);
addr2 = gen_rtx_PLUS (word_mode, src2_addr, adj_reg);
}
else
{
addr1 = src1_addr;
addr2 = src2_addr;
}
do_load_for_compare_from_addr (load_mode, d1, addr1, orig_src1);
do_load_for_compare_from_addr (load_mode, d2, addr2, orig_src2);
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1, d2);
emit_insn (gen_rtx_SET (dcond, cmp));
}
else
{
if (word_mode == DImode)
emit_insn (gen_subfdi3_carry (diff, d2, d1));
else
emit_insn (gen_subfsi3_carry (diff, d2, d1));
}
}
/* Generate the sequence of compares for strcmp/strncmp using vec/vsx
instructions.
BYTES_TO_COMPARE is the number of bytes to be compared.
ORIG_SRC1 is the unmodified rtx for the first string.
ORIG_SRC2 is the unmodified rtx for the second string.
S1ADDR is the register to use for the base address of the first string.
S2ADDR is the register to use for the base address of the second string.
OFF_REG is the register to use for the string offset for loads.
S1DATA is the register for loading the first string.
S2DATA is the register for loading the second string.
VEC_RESULT is the rtx for the vector result indicating the byte difference.
EQUALITY_COMPARE_REST is a flag to indicate we need to make a cleanup call
to strcmp/strncmp if we have equality at the end of the inline comparison.
P_CLEANUP_LABEL is a pointer to rtx for a label we generate if we need code
to clean up and generate the final comparison result.
FINAL_MOVE_LABEL is rtx for a label we can branch to when we can just
set the final result.
CHECKZERO indicates whether the sequence should check for zero bytes
for use doing strncmp, or not (for use doing memcmp). */
static void
expand_cmp_vec_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
rtx orig_src1, rtx orig_src2,
rtx s1addr, rtx s2addr, rtx off_reg,
rtx s1data, rtx s2data, rtx vec_result,
bool equality_compare_rest, rtx *p_cleanup_label,
rtx final_move_label, bool checkzero)
{
machine_mode load_mode;
unsigned int load_mode_size;
unsigned HOST_WIDE_INT cmp_bytes = 0;
unsigned HOST_WIDE_INT offset = 0;
rtx zero_reg = NULL;
gcc_assert (p_cleanup_label != NULL);
rtx cleanup_label = *p_cleanup_label;
emit_move_insn (s1addr, force_reg (Pmode, XEXP (orig_src1, 0)));
emit_move_insn (s2addr, force_reg (Pmode, XEXP (orig_src2, 0)));
if (checkzero && !TARGET_P9_VECTOR)
{
zero_reg = gen_reg_rtx (V16QImode);
emit_move_insn (zero_reg, CONST0_RTX (V16QImode));
}
while (bytes_to_compare > 0)
{
/* VEC/VSX compare sequence for P8:
check each 16B with:
lxvd2x 32,28,8
lxvd2x 33,29,8
vcmpequb 2,0,1 # compare strings
vcmpequb 4,0,3 # compare w/ 0
xxlorc 37,36,34 # first FF byte is either mismatch or end of string
vcmpequb. 7,5,3 # reg 7 contains 0
bnl 6,.Lmismatch
For the P8 LE case, we use lxvd2x and compare full 16 bytes
but then use vgbbd and a shift to get two bytes with the
information we need in the correct order.
VEC/VSX compare sequence if TARGET_P9_VECTOR:
lxvb16x/lxvb16x # load 16B of each string
vcmpnezb. # produces difference location or zero byte location
bne 6,.Lmismatch
Use the overlapping compare trick for the last block if it is
less than 16 bytes.
*/
load_mode = V16QImode;
load_mode_size = GET_MODE_SIZE (load_mode);
if (bytes_to_compare >= load_mode_size)
cmp_bytes = load_mode_size;
else
{
/* Move this load back so it doesn't go past the end. P8/P9
can do this efficiently. This is never called with less
than 16 bytes so we should always be able to do this. */
unsigned int extra_bytes = load_mode_size - bytes_to_compare;
cmp_bytes = bytes_to_compare;
gcc_assert (offset > extra_bytes);
offset -= extra_bytes;
cmp_bytes = load_mode_size;
bytes_to_compare = cmp_bytes;
}
/* The offset currently used is always kept in off_reg so that the
cleanup code on P8 can use it to extract the differing byte. */
emit_move_insn (off_reg, GEN_INT (offset));
rtx addr1 = gen_rtx_PLUS (Pmode, s1addr, off_reg);
do_load_for_compare_from_addr (load_mode, s1data, addr1, orig_src1);
rtx addr2 = gen_rtx_PLUS (Pmode, s2addr, off_reg);
do_load_for_compare_from_addr (load_mode, s2data, addr2, orig_src2);
/* Cases to handle. A and B are chunks of the two strings.
1: Not end of comparison:
A != B: branch to cleanup code to compute result.
A == B: next block
2: End of the inline comparison:
A != B: branch to cleanup code to compute result.
A == B: call strcmp/strncmp
3: compared requested N bytes:
A == B: branch to result 0.
A != B: cleanup code to compute result. */
unsigned HOST_WIDE_INT remain = bytes_to_compare - cmp_bytes;
if (checkzero)
{
if (TARGET_P9_VECTOR)
emit_insn (gen_vcmpnezb_p (vec_result, s1data, s2data));
else
{
/* Emit instructions to do comparison and zero check. */
rtx cmp_res = gen_reg_rtx (load_mode);
rtx cmp_zero = gen_reg_rtx (load_mode);
rtx cmp_combined = gen_reg_rtx (load_mode);
emit_insn (gen_altivec_eqv16qi (cmp_res, s1data, s2data));
emit_insn (gen_altivec_eqv16qi (cmp_zero, s1data, zero_reg));
emit_insn (gen_orcv16qi3 (vec_result, cmp_zero, cmp_res));
emit_insn (gen_altivec_vcmpequb_p (cmp_combined, vec_result, zero_reg));
}
}
else
emit_insn (gen_altivec_vcmpequb_p (vec_result, s1data, s2data));
bool branch_to_cleanup = (remain > 0 || equality_compare_rest);
rtx cr6 = gen_rtx_REG (CCmode, CR6_REGNO);
rtx dst_label;
rtx cmp_rtx;
if (branch_to_cleanup)
{
/* Branch to cleanup code, otherwise fall through to do more
compares. P8 and P9 use different CR bits because on P8
we are looking at the result of a comparsion vs a
register of zeroes so the all-true condition means no
difference or zero was found. On P9, vcmpnezb sets a byte
to 0xff if there is a mismatch or zero, so the all-false
condition indicates we found no difference or zero. */
if (!cleanup_label)
cleanup_label = gen_label_rtx ();
dst_label = cleanup_label;
if (TARGET_P9_VECTOR && checkzero)
cmp_rtx = gen_rtx_NE (VOIDmode, cr6, const0_rtx);
else
cmp_rtx = gen_rtx_GE (VOIDmode, cr6, const0_rtx);
}
else
{
/* Branch to final return or fall through to cleanup,
result is already set to 0. */
dst_label = final_move_label;
if (TARGET_P9_VECTOR && checkzero)
cmp_rtx = gen_rtx_EQ (VOIDmode, cr6, const0_rtx);
else
cmp_rtx = gen_rtx_LT (VOIDmode, cr6, const0_rtx);
}
rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, dst_label);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
lab_ref, pc_rtx);
rtx_insn *j2 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j2, profile_probability::likely ());
JUMP_LABEL (j2) = dst_label;
LABEL_NUSES (dst_label) += 1;
offset += cmp_bytes;
bytes_to_compare -= cmp_bytes;
}
*p_cleanup_label = cleanup_label;
return;
}
/* Generate the final sequence that identifies the differing
byte and generates the final result, taking into account
zero bytes:
P8:
vgbbd 0,0
vsldoi 0,0,0,9
mfvsrd 9,32
addi 10,9,-1 # count trailing zero bits
andc 9,10,9
popcntd 9,9
lbzx 10,28,9 # use that offset to load differing byte
lbzx 3,29,9
subf 3,3,10 # subtract for final result
P9:
vclzlsbb # counts trailing bytes with lsb=0
vextublx # extract differing byte
STR1 is the reg rtx for data from string 1.
STR2 is the reg rtx for data from string 2.
RESULT is the reg rtx for the comparison result.
S1ADDR is the register to use for the base address of the first string.
S2ADDR is the register to use for the base address of the second string.
ORIG_SRC1 is the unmodified rtx for the first string.
ORIG_SRC2 is the unmodified rtx for the second string.
OFF_REG is the register to use for the string offset for loads.
VEC_RESULT is the rtx for the vector result indicating the byte difference. */
static void
emit_final_compare_vec (rtx str1, rtx str2, rtx result,
rtx s1addr, rtx s2addr,
rtx orig_src1, rtx orig_src2,
rtx off_reg, rtx vec_result)
{
if (TARGET_P9_VECTOR)
{
rtx diffix = gen_reg_rtx (SImode);
rtx chr1 = gen_reg_rtx (SImode);
rtx chr2 = gen_reg_rtx (SImode);
rtx chr1_di = simplify_gen_subreg (DImode, chr1, SImode, 0);
rtx chr2_di = simplify_gen_subreg (DImode, chr2, SImode, 0);
emit_insn (gen_vclzlsbb_v16qi (diffix, vec_result));
emit_insn (gen_vextublx (chr1, diffix, str1));
emit_insn (gen_vextublx (chr2, diffix, str2));
do_sub3 (result, chr1_di, chr2_di);
}
else
{
gcc_assert (TARGET_P8_VECTOR);
rtx diffix = gen_reg_rtx (DImode);
rtx result_gbbd = gen_reg_rtx (V16QImode);
/* Since each byte of the input is either 00 or FF, the bytes in
dw0 and dw1 after vgbbd are all identical to each other. */
emit_insn (gen_p8v_vgbbd (result_gbbd, vec_result));
/* For LE, we shift by 9 and get BA in the low two bytes then CTZ.
For BE, we shift by 7 and get AB in the high two bytes then CLZ. */
rtx result_shifted = gen_reg_rtx (V16QImode);
int shift_amt = (BYTES_BIG_ENDIAN) ? 7 : 9;
emit_insn (gen_altivec_vsldoi_v16qi (result_shifted, result_gbbd,
result_gbbd, GEN_INT (shift_amt)));
rtx diffix_df = simplify_gen_subreg (DFmode, diffix, DImode, 0);
emit_insn (gen_p8_mfvsrd_3_v16qi (diffix_df, result_shifted));
rtx count = gen_reg_rtx (DImode);
if (BYTES_BIG_ENDIAN)
emit_insn (gen_clzdi2 (count, diffix));
else
emit_insn (gen_ctzdi2 (count, diffix));
/* P8 doesn't have a good solution for extracting one byte from
a vsx reg like vextublx on P9 so we just compute the offset
of the differing byte and load it from each string. */
do_add3 (off_reg, off_reg, count);
rtx chr1 = gen_reg_rtx (QImode);
rtx chr2 = gen_reg_rtx (QImode);
rtx addr1 = gen_rtx_PLUS (Pmode, s1addr, off_reg);
do_load_for_compare_from_addr (QImode, chr1, addr1, orig_src1);
rtx addr2 = gen_rtx_PLUS (Pmode, s2addr, off_reg);
do_load_for_compare_from_addr (QImode, chr2, addr2, orig_src2);
machine_mode rmode = GET_MODE (result);
rtx chr1_rm = simplify_gen_subreg (rmode, chr1, QImode, 0);
rtx chr2_rm = simplify_gen_subreg (rmode, chr2, QImode, 0);
do_sub3 (result, chr1_rm, chr2_rm);
}
return;
}
/* Expand a block compare operation using loop code, and return true
if successful. Return false if we should let the compiler generate
normal code, probably a memcmp call.
OPERANDS[0] is the target (result).
OPERANDS[1] is the first source.
OPERANDS[2] is the second source.
OPERANDS[3] is the length.
OPERANDS[4] is the alignment. */
bool
expand_compare_loop (rtx operands[])
{
rtx target = operands[0];
rtx orig_src1 = operands[1];
rtx orig_src2 = operands[2];
rtx bytes_rtx = operands[3];
rtx align_rtx = operands[4];
/* This case is complicated to handle because the subtract
with carry instructions do not generate the 64-bit
carry and so we must emit code to calculate it ourselves.
We choose not to implement this yet. */
if (TARGET_32BIT && TARGET_POWERPC64)
return false;
/* Allow non-const length. */
int bytes_is_const = CONST_INT_P (bytes_rtx);
/* This must be a fixed size alignment. */
if (!CONST_INT_P (align_rtx))
return false;
HOST_WIDE_INT align1 = MEM_ALIGN (orig_src1) / BITS_PER_UNIT;
HOST_WIDE_INT align2 = MEM_ALIGN (orig_src2) / BITS_PER_UNIT;
HOST_WIDE_INT minalign = MIN (align1, align2);
bool isP7 = (rs6000_tune == PROCESSOR_POWER7);
gcc_assert (GET_MODE (target) == SImode);
/* Anything to move? */
HOST_WIDE_INT bytes = 0;
if (bytes_is_const)
bytes = INTVAL (bytes_rtx);
if (bytes_is_const && bytes == 0)
return true;
/* Limit the amount we compare, if known statically. */
HOST_WIDE_INT max_bytes;
switch (rs6000_tune)
{
case PROCESSOR_POWER7:
if (!bytes_is_const)
if (minalign < 8)
max_bytes = 0;
else
max_bytes = 128;
else
if (minalign < 8)
max_bytes = 32;
else
max_bytes = 128;
break;
case PROCESSOR_POWER8:
if (!bytes_is_const)
max_bytes = 0;
else
if (minalign < 8)
max_bytes = 128;
else
max_bytes = 64;
break;
case PROCESSOR_POWER9:
case PROCESSOR_POWER10:
if (bytes_is_const)
max_bytes = 191;
else
max_bytes = 0;
break;
default:
max_bytes = 128;
}
/* Allow the option to override the default. */
if (rs6000_block_compare_inline_loop_limit >= 0)
max_bytes = (unsigned HOST_WIDE_INT) rs6000_block_compare_inline_loop_limit;
if (max_bytes == 0)
return false;
rtx cmp_rem = gen_reg_rtx (word_mode); /* Remainder for library call. */
rtx loop_cmp = gen_reg_rtx (word_mode); /* Actual amount compared by loop. */
HOST_WIDE_INT niter;
rtx iter = gen_reg_rtx (word_mode);
rtx iv1 = gen_reg_rtx (word_mode);
rtx iv2 = gen_reg_rtx (word_mode);
rtx d1_1 = gen_reg_rtx (word_mode); /* Addr expression src1+iv1 */
rtx d1_2 = gen_reg_rtx (word_mode); /* Addr expression src1+iv2 */
rtx d2_1 = gen_reg_rtx (word_mode); /* Addr expression src2+iv1 */
rtx d2_2 = gen_reg_rtx (word_mode); /* Addr expression src2+iv2 */
/* Strip unneeded subreg from length if there is one. */
if (SUBREG_P (bytes_rtx) && subreg_lowpart_p (bytes_rtx))
bytes_rtx = SUBREG_REG (bytes_rtx);
/* Extend bytes_rtx to word_mode if needed. But, we expect only to
maybe have to deal with the case were bytes_rtx is SImode and
word_mode is DImode. */
if (!bytes_is_const)
{
if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) > GET_MODE_SIZE (word_mode))
/* Do not expect length longer than word_mode. */
return false;
else if (GET_MODE_SIZE (GET_MODE (bytes_rtx)) < GET_MODE_SIZE (word_mode))
{
bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
bytes_rtx = force_reg (word_mode,
gen_rtx_fmt_e (ZERO_EXTEND, word_mode,
bytes_rtx));
}
else
/* Make sure it's in a register before we get started. */
bytes_rtx = force_reg (GET_MODE (bytes_rtx), bytes_rtx);
}
machine_mode load_mode = word_mode;
HOST_WIDE_INT load_mode_size = GET_MODE_SIZE (load_mode);
/* Number of bytes per iteration of the unrolled loop. */
HOST_WIDE_INT loop_bytes = 2 * load_mode_size;
/* max iters and bytes compared in the loop. */
HOST_WIDE_INT max_loop_iter = max_bytes / loop_bytes;
HOST_WIDE_INT max_loop_bytes = max_loop_iter * loop_bytes;
int l2lb = floor_log2 (loop_bytes);
if (bytes_is_const && (max_bytes < load_mode_size
|| !IN_RANGE (bytes, load_mode_size, max_bytes)))
return false;
bool no_remainder_code = false;
rtx final_label = gen_label_rtx ();
rtx final_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
rtx diff_label = gen_label_rtx ();
rtx library_call_label = NULL;
rtx cleanup_label = gen_label_rtx ();
rtx cr;
rtx src1_addr = copy_addr_to_reg (XEXP (orig_src1, 0));
rtx src2_addr = copy_addr_to_reg (XEXP (orig_src2, 0));
/* Difference found is stored here before jump to diff_label. */
rtx diff = gen_reg_rtx (word_mode);
rtx_insn *j;
/* Example of generated code for 35 bytes aligned 1 byte.
mtctr 8
li 6,0
li 5,8
.L13:
ldbrx 7,3,6
ldbrx 9,10,6
ldbrx 0,3,5
ldbrx 4,10,5
addi 6,6,16
addi 5,5,16
subfc. 9,9,7
bne 0,.L10
subfc. 9,4,0
bdnzt 2,.L13
bne 0,.L10
add 3,3,6
add 10,10,6
addi 9,3,-5
ldbrx 7,0,9
addi 9,10,-5
ldbrx 9,0,9
subfc 9,9,7
.p2align 4,,15
.L10:
popcntd 9,9
subfe 10,10,10
or 9,9,10
Compiled with -fno-reorder-blocks for clarity. */
/* Structure of what we're going to do:
Two separate lengths: what we will compare before bailing to library
call (max_bytes), and the total length to be checked.
if length <= 16, branch to linear cleanup code starting with
remainder length check (length not known at compile time)
set up 2 iv's and load count reg, compute remainder length
unrollx2 compare loop
if loop exit due to a difference, branch to difference handling code
if remainder length < 8, branch to final cleanup compare
load and compare 8B
final cleanup comparison (depends on alignment and length)
load 8B, shift off bytes past length, compare
load 8B ending at last byte and compare
load/compare 1 byte at a time (short block abutting 4k boundary)
difference handling, 64->32 conversion
final result
branch around memcmp call
memcmp library call
*/
/* If bytes is not const, compare length and branch directly
to the cleanup code that can handle 0-16 bytes if length
is >= 16. Stash away bytes-max_bytes for the library call. */
if (bytes_is_const)
{
/* These need to be set for some of the places we may jump to. */
if (bytes > max_bytes)
{
no_remainder_code = true;
niter = max_loop_iter;
library_call_label = gen_label_rtx ();
}
else
{
niter = bytes / loop_bytes;
}
emit_move_insn (iter, GEN_INT (niter));
emit_move_insn (loop_cmp, GEN_INT (niter * loop_bytes));
emit_move_insn (cmp_rem, GEN_INT (bytes - niter * loop_bytes));
}
else
{
library_call_label = gen_label_rtx ();
/* If we go to the cleanup code, it expects length to be in cmp_rem. */
emit_move_insn (cmp_rem, bytes_rtx);
/* Check for > max_bytes bytes. We want to bail out as quickly as
possible if we have to go over to memcmp. */
do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (max_bytes),
NULL_RTX, library_call_label, profile_probability::even ());
/* Check for < loop_bytes bytes. */
do_ifelse (CCmode, LT, bytes_rtx, GEN_INT (loop_bytes),
NULL_RTX, cleanup_label, profile_probability::even ());
/* Loop compare bytes and iterations if bytes>max_bytes. */
rtx mb_reg = gen_reg_rtx (word_mode);
emit_move_insn (mb_reg, GEN_INT (max_loop_bytes));
rtx mi_reg = gen_reg_rtx (word_mode);
emit_move_insn (mi_reg, GEN_INT (max_loop_iter));
/* Compute number of loop iterations if bytes <= max_bytes. */
if (word_mode == DImode)
emit_insn (gen_lshrdi3 (iter, bytes_rtx, GEN_INT (l2lb)));
else
emit_insn (gen_lshrsi3 (iter, bytes_rtx, GEN_INT (l2lb)));
/* Compute bytes to compare in loop if bytes <= max_bytes. */
rtx mask = GEN_INT (HOST_WIDE_INT_M1U << l2lb);
if (word_mode == DImode)
{
emit_insn (gen_anddi3 (loop_cmp, bytes_rtx, mask));
}
else
{
emit_insn (gen_andsi3 (loop_cmp, bytes_rtx, mask));
}
/* Check for bytes <= max_bytes. */
if (TARGET_ISEL)
{
/* P9 has fast isel so we use one compare and two isel. */
cr = gen_reg_rtx (CCmode);
rtx compare_rtx = gen_rtx_COMPARE (CCmode, bytes_rtx,
GEN_INT (max_bytes));
emit_move_insn (cr, compare_rtx);
rtx cmp_rtx = gen_rtx_LE (VOIDmode, cr, const0_rtx);
do_isel (loop_cmp, cmp_rtx, loop_cmp, mb_reg, cr);
do_isel (iter, cmp_rtx, iter, mi_reg, cr);
}
else
{
rtx lab_after = gen_label_rtx ();
do_ifelse (CCmode, LE, bytes_rtx, GEN_INT (max_bytes),
NULL_RTX, lab_after, profile_probability::even ());
emit_move_insn (loop_cmp, mb_reg);
emit_move_insn (iter, mi_reg);
emit_label (lab_after);
}
/* Now compute remainder bytes which isn't used until after the loop. */
do_sub3 (cmp_rem, bytes_rtx, loop_cmp);
}
rtx dcond = NULL_RTX; /* Used for when we jump to diff_label. */
/* For p9 we need to have just one of these as multiple places define
it and it gets used by the setb at the end. */
if (TARGET_P9_MISC)
dcond = gen_reg_rtx (CCUNSmode);
if (!bytes_is_const || bytes >= loop_bytes)
{
/* It should not be possible to come here if remaining bytes is
< 16 in the runtime case either. Compute number of loop
iterations. We compare 2*word_mode per iteration so 16B for
64-bit code and 8B for 32-bit. Set up two induction
variables and load count register. */
/* HACK ALERT: create hard reg for CTR here. If we just use a
pseudo, cse will get rid of it and then the allocator will
see it used in the lshr above and won't give us ctr. */
rtx ctr = gen_rtx_REG (Pmode, CTR_REGNO);
emit_move_insn (ctr, iter);
emit_move_insn (diff, GEN_INT (0));
emit_move_insn (iv1, GEN_INT (0));
emit_move_insn (iv2, GEN_INT (load_mode_size));
/* inner loop to compare 2*word_mode */
rtx loop_top_label = gen_label_rtx ();
emit_label (loop_top_label);
rtx src1_ix1 = gen_rtx_PLUS (word_mode, src1_addr, iv1);
rtx src2_ix1 = gen_rtx_PLUS (word_mode, src2_addr, iv1);
do_load_for_compare_from_addr (load_mode, d1_1,
src1_ix1, orig_src1);
do_load_for_compare_from_addr (load_mode, d2_1,
src2_ix1, orig_src2);
do_add3 (iv1, iv1, GEN_INT (loop_bytes));
rtx src1_ix2 = gen_rtx_PLUS (word_mode, src1_addr, iv2);
rtx src2_ix2 = gen_rtx_PLUS (word_mode, src2_addr, iv2);
do_load_for_compare_from_addr (load_mode, d1_2,
src1_ix2, orig_src1);
do_load_for_compare_from_addr (load_mode, d2_2,
src2_ix2, orig_src2);
do_add3 (iv2, iv2, GEN_INT (loop_bytes));
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_1, d2_1);
emit_insn (gen_rtx_SET (dcond, cmp));
}
else
{
dcond = gen_reg_rtx (CCmode);
if (word_mode == DImode)
emit_insn (gen_subfdi3_carry_dot2 (diff, d2_1, d1_1, dcond));
else
emit_insn (gen_subfsi3_carry_dot2 (diff, d2_1, d1_1, dcond));
}
do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
dcond, diff_label, profile_probability::unlikely ());
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_2, d2_2);
emit_insn (gen_rtx_SET (dcond, cmp));
}
else
{
dcond = gen_reg_rtx (CCmode);
if (word_mode == DImode)
emit_insn (gen_subfdi3_carry_dot2 (diff, d2_2, d1_2, dcond));
else
emit_insn (gen_subfsi3_carry_dot2 (diff, d2_2, d1_2, dcond));
}
rtx eqrtx = gen_rtx_EQ (VOIDmode, d1_2, d2_2);
if (TARGET_64BIT)
j = emit_jump_insn (gen_bdnztf_di (loop_top_label, ctr, ctr,
eqrtx, dcond));
else
j = emit_jump_insn (gen_bdnztf_si (loop_top_label, ctr, ctr,
eqrtx, dcond));
add_reg_br_prob_note (j, profile_probability::likely ());
JUMP_LABEL (j) = loop_top_label;
LABEL_NUSES (loop_top_label) += 1;
}
HOST_WIDE_INT bytes_remaining = 0;
if (bytes_is_const)
bytes_remaining = (bytes % loop_bytes);
/* If diff is nonzero, branch to difference handling
code. If we exit here with a nonzero diff, it is
because the second word differed. */
if (TARGET_P9_MISC)
do_ifelse (CCUNSmode, NE, NULL_RTX, NULL_RTX, dcond,
diff_label, profile_probability::unlikely ());
else
do_ifelse (CCmode, NE, diff, const0_rtx, NULL_RTX,
diff_label, profile_probability::unlikely ());
if (library_call_label != NULL && bytes_is_const && bytes > max_bytes)
{
/* If the length is known at compile time, then we will always
have a remainder to go to the library call with. */
rtx library_call_ref = gen_rtx_LABEL_REF (VOIDmode, library_call_label);
j = emit_jump_insn (gen_rtx_SET (pc_rtx, library_call_ref));
JUMP_LABEL (j) = library_call_label;
LABEL_NUSES (library_call_label) += 1;
emit_barrier ();
}
if (bytes_is_const && bytes_remaining == 0)
{
/* No remainder and if we are here then diff is 0 so just return 0 */
if (TARGET_64BIT)
emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
else
emit_move_insn (target, diff);
j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
JUMP_LABEL (j) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
}
else if (!no_remainder_code)
{
/* Update addresses to point to the next word to examine. */
do_add3 (src1_addr, src1_addr, iv1);
do_add3 (src2_addr, src2_addr, iv1);
emit_label (cleanup_label);
if (!bytes_is_const)
{
/* If we're dealing with runtime length, we have to check if
it's zero after the loop. When length is known at compile
time the no-remainder condition is dealt with above. By
doing this after cleanup_label, we also deal with the
case where length is 0 at the start and we bypass the
loop with a branch to cleanup_label. */
emit_move_insn (target, const0_rtx);
do_ifelse (CCmode, EQ, cmp_rem, const0_rtx,
NULL_RTX, final_label, profile_probability::unlikely ());
}
rtx final_cleanup = gen_label_rtx ();
rtx cmp_rem_before = gen_reg_rtx (word_mode);
/* Compare one more word_mode chunk if needed. */
if (!bytes_is_const || bytes_remaining >= load_mode_size)
{
/* If remainder length < word length, branch to final
cleanup compare. */
if (!bytes_is_const)
{
do_ifelse (CCmode, LT, cmp_rem, GEN_INT (load_mode_size),
NULL_RTX, final_cleanup, profile_probability::even ());
}
/* load and compare 8B */
do_load_for_compare_from_addr (load_mode, d1_1,
src1_addr, orig_src1);
do_load_for_compare_from_addr (load_mode, d2_1,
src2_addr, orig_src2);
/* Compare the word, see if we need to do the last partial. */
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, d1_1, d2_1);
emit_insn (gen_rtx_SET (dcond, cmp));
}
else
{
dcond = gen_reg_rtx (CCmode);
if (word_mode == DImode)
emit_insn (gen_subfdi3_carry_dot2 (diff, d2_1, d1_1, dcond));
else
emit_insn (gen_subfsi3_carry_dot2 (diff, d2_1, d1_1, dcond));
}
do_ifelse (GET_MODE (dcond), NE, NULL_RTX, NULL_RTX,
dcond, diff_label, profile_probability::even ());
do_add3 (src1_addr, src1_addr, GEN_INT (load_mode_size));
do_add3 (src2_addr, src2_addr, GEN_INT (load_mode_size));
emit_move_insn (cmp_rem_before, cmp_rem);
do_add3 (cmp_rem, cmp_rem, GEN_INT (-load_mode_size));
if (bytes_is_const)
bytes_remaining -= load_mode_size;
else
/* See if remaining length is now zero. We previously set
target to 0 so we can just jump to the end. */
do_ifelse (CCmode, EQ, cmp_rem, const0_rtx, NULL_RTX,
final_label, profile_probability::unlikely ());
}
/* Cases:
bytes_is_const
We can always shift back to do an overlapping compare
of the last chunk because we know length >= 8.
!bytes_is_const
align>=load_mode_size
Read word_mode and mask
align= load_mode_size && align2 >= load_mode_size)
{
/* Alignment is larger than word_mode so we do not need to be
concerned with extra page crossings. But, we do not know
that the length is larger than load_mode_size so we might
end up compareing against data before the block if we try
an overlapping compare. Also we use this on P7 for fixed length
remainder because P7 doesn't like overlapping unaligned.
Strategy: load 8B, shift off bytes past length, and compare. */
emit_label (final_cleanup);
do_load_mask_compare (load_mode, diff, cmp_rem, dcond,
src1_addr, src2_addr, orig_src1, orig_src2);
}
else if (bytes_remaining && bytes_is_const)
{
/* We do not do loop expand if length < 32 so we know at the
end we can do an overlapping compare.
Strategy: shift address back and do word_mode load that
ends at the end of the block. */
emit_label (final_cleanup);
do_overlap_load_compare (load_mode, true, bytes_remaining, diff,
cmp_rem, dcond, src1_addr, src2_addr,
orig_src1, orig_src2);
}
else if (!bytes_is_const)
{
rtx handle4k_label = gen_label_rtx ();
rtx nonconst_overlap = gen_label_rtx ();
emit_label (nonconst_overlap);
/* Here we have to handle the case where whe have runtime
length which may be too short for overlap compare, and
alignment is not at least load_mode_size so we have to
tread carefully to avoid stepping across 4k boundaries. */
/* If the length after the loop was larger than word_mode
size, we can just do an overlapping compare and we're
done. We fall through to this code from the word_mode
compare that preceeds this. */
do_overlap_load_compare (load_mode, false, 0, diff,
cmp_rem, dcond, src1_addr, src2_addr,
orig_src1, orig_src2);
rtx diff_ref = gen_rtx_LABEL_REF (VOIDmode, diff_label);
j = emit_jump_insn (gen_rtx_SET (pc_rtx, diff_ref));
JUMP_LABEL (j) = diff_label;
LABEL_NUSES (diff_label) += 1;
emit_barrier ();
/* If we couldn't do the overlap compare we have to be more
careful of the 4k boundary. Test to see if either
address is less than word_mode_size away from a 4k
boundary. If not, then we can do a load/shift/compare
and we are done. We come to this code if length was less
than word_mode_size. */
emit_label (final_cleanup);
/* We can still avoid the slow case if the length was larger
than one loop iteration, in which case go do the overlap
load compare path. */
do_ifelse (CCmode, GT, bytes_rtx, GEN_INT (loop_bytes),
NULL_RTX, nonconst_overlap, profile_probability::even ());
rtx rem4k = gen_reg_rtx (word_mode);
rtx dist1 = gen_reg_rtx (word_mode);
rtx dist2 = gen_reg_rtx (word_mode);
do_sub3 (rem4k, GEN_INT (4096), cmp_rem);
if (word_mode == SImode)
emit_insn (gen_andsi3 (dist1, src1_addr, GEN_INT (0xfff)));
else
emit_insn (gen_anddi3 (dist1, src1_addr, GEN_INT (0xfff)));
do_ifelse (CCmode, LE, dist1, rem4k, NULL_RTX,
handle4k_label, profile_probability::very_unlikely ());
if (word_mode == SImode)
emit_insn (gen_andsi3 (dist2, src2_addr, GEN_INT (0xfff)));
else
emit_insn (gen_anddi3 (dist2, src2_addr, GEN_INT (0xfff)));
do_ifelse (CCmode, LE, dist2, rem4k, NULL_RTX,
handle4k_label, profile_probability::very_unlikely ());
/* We don't have a 4k boundary to deal with, so do
a load/shift/compare and jump to diff. */
do_load_mask_compare (load_mode, diff, cmp_rem, dcond,
src1_addr, src2_addr, orig_src1, orig_src2);
j = emit_jump_insn (gen_rtx_SET (pc_rtx, diff_ref));
JUMP_LABEL (j) = diff_label;
LABEL_NUSES (diff_label) += 1;
emit_barrier ();
/* Finally in the unlikely case we are inching up to a
4k boundary we use a compact lbzx/compare loop to do
it a byte at a time. */
emit_label (handle4k_label);
rtx ctr = gen_rtx_REG (Pmode, CTR_REGNO);
emit_move_insn (ctr, cmp_rem);
rtx ixreg = gen_reg_rtx (Pmode);
emit_move_insn (ixreg, const0_rtx);
rtx src1_ix = gen_rtx_PLUS (word_mode, src1_addr, ixreg);
rtx src2_ix = gen_rtx_PLUS (word_mode, src2_addr, ixreg);
rtx d1 = gen_reg_rtx (word_mode);
rtx d2 = gen_reg_rtx (word_mode);
rtx fc_loop = gen_label_rtx ();
emit_label (fc_loop);
do_load_for_compare_from_addr (QImode, d1, src1_ix, orig_src1);
do_load_for_compare_from_addr (QImode, d2, src2_ix, orig_src2);
do_add3 (ixreg, ixreg, const1_rtx);
rtx cond = gen_reg_rtx (CCmode);
rtx subexpr = gen_rtx_MINUS (word_mode, d1, d2);
rs6000_emit_dot_insn (diff, subexpr, 2, cond);
rtx eqrtx = gen_rtx_EQ (VOIDmode, d1, d2);
if (TARGET_64BIT)
j = emit_jump_insn (gen_bdnztf_di (fc_loop, ctr, ctr,
eqrtx, cond));
else
j = emit_jump_insn (gen_bdnztf_si (fc_loop, ctr, ctr,
eqrtx, cond));
add_reg_br_prob_note (j, profile_probability::likely ());
JUMP_LABEL (j) = fc_loop;
LABEL_NUSES (fc_loop) += 1;
if (TARGET_64BIT)
emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
else
emit_move_insn (target, diff);
/* Since we are comparing bytes, the difference can be used
as the final result and we are done here. */
j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
JUMP_LABEL (j) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
}
}
emit_label (diff_label);
/* difference handling, 64->32 conversion */
/* We need to produce DI result from sub, then convert to target SI
while maintaining <0 / ==0 / >0 properties. This sequence works:
subfc L,A,B
subfe H,H,H
popcntd L,L
rldimi L,H,6,0
This is an alternate one Segher cooked up if somebody
wants to expand this for something that doesn't have popcntd:
subfc L,a,b
subfe H,x,x
addic t,L,-1
subfe v,t,L
or z,v,H
And finally, p9 can just do this:
cmpld A,B
setb r */
if (TARGET_P9_MISC)
emit_insn (gen_setb_unsigned (target, dcond));
else
{
if (TARGET_64BIT)
{
rtx tmp_reg_ca = gen_reg_rtx (DImode);
emit_insn (gen_subfdi3_carry_in_xx (tmp_reg_ca));
emit_insn (gen_popcntddi2 (diff, diff));
emit_insn (gen_iordi3 (diff, diff, tmp_reg_ca));
emit_insn (gen_movsi (target, gen_lowpart (SImode, diff)));
}
else
{
rtx tmp_reg_ca = gen_reg_rtx (SImode);
emit_insn (gen_subfsi3_carry_in_xx (tmp_reg_ca));
emit_insn (gen_popcntdsi2 (diff, diff));
emit_insn (gen_iorsi3 (target, diff, tmp_reg_ca));
}
}
if (library_call_label != NULL)
{
/* Branch around memcmp call. */
j = emit_jump_insn (gen_rtx_SET (pc_rtx, final_ref));
JUMP_LABEL (j) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
/* Make memcmp library call. cmp_rem is the remaining bytes that
were compared and cmp_rem is the expected amount to be compared
by memcmp. If we don't find a difference in the loop compare, do
the library call directly instead of doing a small compare just
to get to an arbitrary boundary before calling it anyway.
Also, update addresses to point to the next word to examine. */
emit_label (library_call_label);
rtx len_rtx = gen_reg_rtx (word_mode);
if (bytes_is_const)
{
emit_move_insn (len_rtx, cmp_rem);
do_add3 (src1_addr, src1_addr, iv1);
do_add3 (src2_addr, src2_addr, iv1);
}
else
emit_move_insn (len_rtx, bytes_rtx);
tree fun = builtin_decl_explicit (BUILT_IN_MEMCMP);
emit_library_call_value (XEXP (DECL_RTL (fun), 0),
target, LCT_NORMAL, GET_MODE (target),
src1_addr, Pmode,
src2_addr, Pmode,
len_rtx, GET_MODE (len_rtx));
}
/* emit final_label */
emit_label (final_label);
return true;
}
/* Generate code to convert a DImode-plus-carry subtract result into
a SImode result that has the same <0 / ==0 / >0 properties to
produce the final result from memcmp.
TARGET is the rtx for the register to receive the memcmp result.
SUB_RESULT is the rtx for the register contining the subtract result. */
void
generate_6432_conversion(rtx target, rtx sub_result)
{
/* We need to produce DI result from sub, then convert to target SI
while maintaining <0 / ==0 / >0 properties. This sequence works:
subfc L,A,B
subfe H,H,H
popcntd L,L
rldimi L,H,6,0
This is an alternate one Segher cooked up if somebody
wants to expand this for something that doesn't have popcntd:
subfc L,a,b
subfe H,x,x
addic t,L,-1
subfe v,t,L
or z,v,H
And finally, p9 can just do this:
cmpld A,B
setb r */
if (TARGET_64BIT)
{
rtx tmp_reg_ca = gen_reg_rtx (DImode);
emit_insn (gen_subfdi3_carry_in_xx (tmp_reg_ca));
rtx popcnt = gen_reg_rtx (DImode);
emit_insn (gen_popcntddi2 (popcnt, sub_result));
rtx tmp2 = gen_reg_rtx (DImode);
emit_insn (gen_iordi3 (tmp2, popcnt, tmp_reg_ca));
emit_insn (gen_movsi (target, gen_lowpart (SImode, tmp2)));
}
else
{
rtx tmp_reg_ca = gen_reg_rtx (SImode);
emit_insn (gen_subfsi3_carry_in_xx (tmp_reg_ca));
rtx popcnt = gen_reg_rtx (SImode);
emit_insn (gen_popcntdsi2 (popcnt, sub_result));
emit_insn (gen_iorsi3 (target, popcnt, tmp_reg_ca));
}
}
/* Generate memcmp expansion using in-line non-loop GPR instructions.
The bool return indicates whether code for a 64->32 conversion
should be generated.
BYTES is the number of bytes to be compared.
BASE_ALIGN is the minimum alignment for both blocks to compare.
ORIG_SRC1 is the original pointer to the first block to compare.
ORIG_SRC2 is the original pointer to the second block to compare.
SUB_RESULT is the reg rtx for the result from the final subtract.
COND is rtx for a condition register that will be used for the final
compare on power9 or better.
FINAL_RESULT is the reg rtx for the final memcmp result.
P_CONVERT_LABEL is a pointer to rtx that will be used to store the
label generated for a branch to the 64->32 code, if such a branch
is needed.
P_FINAL_LABEL is a pointer to rtx that will be used to store the label
for the end of the memcmp if a branch there is needed.
*/
bool
expand_block_compare_gpr(unsigned HOST_WIDE_INT bytes, unsigned int base_align,
rtx orig_src1, rtx orig_src2,
rtx sub_result, rtx cond, rtx final_result,
rtx *p_convert_label, rtx *p_final_label)
{
/* Example of generated code for 18 bytes aligned 1 byte.
Compiled with -fno-reorder-blocks for clarity.
ldbrx 10,31,8
ldbrx 9,7,8
subfc. 9,9,10
bne 0,.L6487
addi 9,12,8
addi 5,11,8
ldbrx 10,0,9
ldbrx 9,0,5
subfc. 9,9,10
bne 0,.L6487
addi 9,12,16
lhbrx 10,0,9
addi 9,11,16
lhbrx 9,0,9
subf 9,9,10
b .L6488
.p2align 4,,15
.L6487: #convert_label
popcntd 9,9
subfe 10,10,10
or 9,9,10
.L6488: #final_label
extsw 10,9
We start off with DImode for two blocks that jump to the DI->SI conversion
if the difference is found there, then a final block of HImode that skips
the DI->SI conversion. */
unsigned HOST_WIDE_INT offset = 0;
unsigned int load_mode_size;
HOST_WIDE_INT cmp_bytes = 0;
rtx src1 = orig_src1;
rtx src2 = orig_src2;
rtx tmp_reg_src1 = gen_reg_rtx (word_mode);
rtx tmp_reg_src2 = gen_reg_rtx (word_mode);
bool need_6432_conv = false;
rtx convert_label = NULL;
rtx final_label = NULL;
machine_mode load_mode;
while (bytes > 0)
{
unsigned int align = compute_current_alignment (base_align, offset);
load_mode = select_block_compare_mode (offset, bytes, align);
load_mode_size = GET_MODE_SIZE (load_mode);
if (bytes >= load_mode_size)
cmp_bytes = load_mode_size;
else if (TARGET_EFFICIENT_OVERLAPPING_UNALIGNED)
{
/* Move this load back so it doesn't go past the end.
P8/P9 can do this efficiently. */
unsigned int extra_bytes = load_mode_size - bytes;
cmp_bytes = bytes;
if (extra_bytes < offset)
{
offset -= extra_bytes;
cmp_bytes = load_mode_size;
bytes = cmp_bytes;
}
}
else
/* P7 and earlier can't do the overlapping load trick fast,
so this forces a non-overlapping load and a shift to get
rid of the extra bytes. */
cmp_bytes = bytes;
src1 = adjust_address (orig_src1, load_mode, offset);
src2 = adjust_address (orig_src2, load_mode, offset);
if (!REG_P (XEXP (src1, 0)))
{
rtx src1_reg = copy_addr_to_reg (XEXP (src1, 0));
src1 = replace_equiv_address (src1, src1_reg);
}
set_mem_size (src1, load_mode_size);
if (!REG_P (XEXP (src2, 0)))
{
rtx src2_reg = copy_addr_to_reg (XEXP (src2, 0));
src2 = replace_equiv_address (src2, src2_reg);
}
set_mem_size (src2, load_mode_size);
do_load_for_compare (tmp_reg_src1, src1, load_mode);
do_load_for_compare (tmp_reg_src2, src2, load_mode);
if (cmp_bytes < load_mode_size)
{
/* Shift unneeded bytes off. */
rtx sh = GEN_INT (BITS_PER_UNIT * (load_mode_size - cmp_bytes));
if (word_mode == DImode)
{
emit_insn (gen_lshrdi3 (tmp_reg_src1, tmp_reg_src1, sh));
emit_insn (gen_lshrdi3 (tmp_reg_src2, tmp_reg_src2, sh));
}
else
{
emit_insn (gen_lshrsi3 (tmp_reg_src1, tmp_reg_src1, sh));
emit_insn (gen_lshrsi3 (tmp_reg_src2, tmp_reg_src2, sh));
}
}
int remain = bytes - cmp_bytes;
if (GET_MODE_SIZE (GET_MODE (final_result)) > GET_MODE_SIZE (load_mode))
{
/* Final_result is larger than load size so we don't need to
reduce result size. */
/* We previously did a block that need 64->32 conversion but
the current block does not, so a label is needed to jump
to the end. */
if (need_6432_conv && !final_label)
final_label = gen_label_rtx ();
if (remain > 0)
{
/* This is not the last block, branch to the end if the result
of this subtract is not zero. */
if (!final_label)
final_label = gen_label_rtx ();
rtx fin_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
rtx tmp = gen_rtx_MINUS (word_mode, tmp_reg_src1, tmp_reg_src2);
rtx cr = gen_reg_rtx (CCmode);
rs6000_emit_dot_insn (tmp_reg_src2, tmp, 2, cr);
emit_insn (gen_movsi (final_result,
gen_lowpart (SImode, tmp_reg_src2)));
rtx ne_rtx = gen_rtx_NE (VOIDmode, cr, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, ne_rtx,
fin_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, profile_probability::unlikely ());
JUMP_LABEL (j) = final_label;
LABEL_NUSES (final_label) += 1;
}
else
{
if (word_mode == DImode)
{
emit_insn (gen_subdi3 (tmp_reg_src2, tmp_reg_src1,
tmp_reg_src2));
emit_insn (gen_movsi (final_result,
gen_lowpart (SImode, tmp_reg_src2)));
}
else
emit_insn (gen_subsi3 (final_result, tmp_reg_src1, tmp_reg_src2));
if (final_label)
{
rtx fin_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, fin_ref));
JUMP_LABEL (j) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
}
}
}
else
{
/* Do we need a 64->32 conversion block? We need the 64->32
conversion even if final_result size == load_mode size because
the subtract generates one extra bit. */
need_6432_conv = true;
if (remain > 0)
{
if (!convert_label)
convert_label = gen_label_rtx ();
/* Compare to zero and branch to convert_label if not zero. */
rtx cvt_ref = gen_rtx_LABEL_REF (VOIDmode, convert_label);
if (TARGET_P9_MISC)
{
/* Generate a compare, and convert with a setb later.
Use cond that is passed in because the caller needs
to use it for the 64->32 conversion later. */
rtx cmp = gen_rtx_COMPARE (CCUNSmode, tmp_reg_src1,
tmp_reg_src2);
emit_insn (gen_rtx_SET (cond, cmp));
}
else
{
/* Generate a subfc. and use the longer sequence for
conversion. Cond is not used outside this
function in this case. */
cond = gen_reg_rtx (CCmode);
if (TARGET_64BIT)
emit_insn (gen_subfdi3_carry_dot2 (sub_result, tmp_reg_src2,
tmp_reg_src1, cond));
else
emit_insn (gen_subfsi3_carry_dot2 (sub_result, tmp_reg_src2,
tmp_reg_src1, cond));
}
rtx ne_rtx = gen_rtx_NE (VOIDmode, cond, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, ne_rtx,
cvt_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, profile_probability::likely ());
JUMP_LABEL (j) = convert_label;
LABEL_NUSES (convert_label) += 1;
}
else
{
/* Just do the subtract/compare. Since this is the last block
the convert code will be generated immediately following. */
if (TARGET_P9_MISC)
{
rtx cmp = gen_rtx_COMPARE (CCUNSmode, tmp_reg_src1,
tmp_reg_src2);
emit_insn (gen_rtx_SET (cond, cmp));
}
else
if (TARGET_64BIT)
emit_insn (gen_subfdi3_carry (sub_result, tmp_reg_src2,
tmp_reg_src1));
else
emit_insn (gen_subfsi3_carry (sub_result, tmp_reg_src2,
tmp_reg_src1));
}
}
offset += cmp_bytes;
bytes -= cmp_bytes;
}
if (convert_label)
*p_convert_label = convert_label;
if (final_label)
*p_final_label = final_label;
return need_6432_conv;
}
/* Expand a block compare operation, and return true if successful.
Return false if we should let the compiler generate normal code,
probably a memcmp call.
OPERANDS[0] is the target (result).
OPERANDS[1] is the first source.
OPERANDS[2] is the second source.
OPERANDS[3] is the length.
OPERANDS[4] is the alignment. */
bool
expand_block_compare (rtx operands[])
{
rtx target = operands[0];
rtx orig_src1 = operands[1];
rtx orig_src2 = operands[2];
rtx bytes_rtx = operands[3];
rtx align_rtx = operands[4];
/* This case is complicated to handle because the subtract
with carry instructions do not generate the 64-bit
carry and so we must emit code to calculate it ourselves.
We choose not to implement this yet. */
if (TARGET_32BIT && TARGET_POWERPC64)
return false;
bool isP7 = (rs6000_tune == PROCESSOR_POWER7);
/* Allow this param to shut off all expansion. */
if (rs6000_block_compare_inline_limit == 0)
return false;
/* targetm.slow_unaligned_access -- don't do unaligned stuff.
However slow_unaligned_access returns true on P7 even though the
performance of this code is good there. */
if (!isP7
&& (targetm.slow_unaligned_access (word_mode, MEM_ALIGN (orig_src1))
|| targetm.slow_unaligned_access (word_mode, MEM_ALIGN (orig_src2))))
return false;
/* Unaligned l*brx traps on P7 so don't do this. However this should
not affect much because LE isn't really supported on P7 anyway. */
if (isP7 && !BYTES_BIG_ENDIAN)
return false;
/* If this is not a fixed size compare, try generating loop code and
if that fails just call memcmp. */
if (!CONST_INT_P (bytes_rtx))
return expand_compare_loop (operands);
/* This must be a fixed size alignment. */
if (!CONST_INT_P (align_rtx))
return false;
unsigned int base_align = UINTVAL (align_rtx) / BITS_PER_UNIT;
gcc_assert (GET_MODE (target) == SImode);
/* Anything to move? */
unsigned HOST_WIDE_INT bytes = UINTVAL (bytes_rtx);
if (bytes == 0)
return true;
/* P7/P8 code uses cond for subfc. but P9 uses
it for cmpld which needs CCUNSmode. */
rtx cond = NULL;
if (TARGET_P9_MISC)
cond = gen_reg_rtx (CCUNSmode);
/* Is it OK to use vec/vsx for this. TARGET_VSX means we have at
least POWER7 but we use TARGET_EFFICIENT_UNALIGNED_VSX which is
at least POWER8. That way we can rely on overlapping compares to
do the final comparison of less than 16 bytes. Also I do not
want to deal with making this work for 32 bits. In addition, we
have to make sure that we have at least P8_VECTOR (we don't allow
P9_VECTOR without P8_VECTOR). */
int use_vec = (bytes >= 33 && !TARGET_32BIT
&& TARGET_EFFICIENT_UNALIGNED_VSX && TARGET_P8_VECTOR);
/* We don't want to generate too much code. The loop code can take
over for lengths greater than 31 bytes. */
unsigned HOST_WIDE_INT max_bytes = rs6000_block_compare_inline_limit;
/* Don't generate too much code if vsx was disabled. */
if (!use_vec && max_bytes > 1)
max_bytes = ((max_bytes + 1) / 2) - 1;
if (!IN_RANGE (bytes, 1, max_bytes))
return expand_compare_loop (operands);
/* The code generated for p7 and older is not faster than glibc
memcmp if alignment is small and length is not short, so bail
out to avoid those conditions. */
if (!TARGET_EFFICIENT_OVERLAPPING_UNALIGNED
&& ((base_align == 1 && bytes > 16)
|| (base_align == 2 && bytes > 32)))
return false;
rtx final_label = NULL;
if (use_vec)
{
rtx final_move_label = gen_label_rtx ();
rtx s1addr = gen_reg_rtx (Pmode);
rtx s2addr = gen_reg_rtx (Pmode);
rtx off_reg = gen_reg_rtx (Pmode);
rtx cleanup_label = NULL;
rtx vec_result = gen_reg_rtx (V16QImode);
rtx s1data = gen_reg_rtx (V16QImode);
rtx s2data = gen_reg_rtx (V16QImode);
rtx result_reg = gen_reg_rtx (word_mode);
emit_move_insn (result_reg, GEN_INT (0));
expand_cmp_vec_sequence (bytes, orig_src1, orig_src2,
s1addr, s2addr, off_reg, s1data, s2data,
vec_result, false,
&cleanup_label, final_move_label, false);
if (cleanup_label)
emit_label (cleanup_label);
emit_insn (gen_one_cmplv16qi2 (vec_result, vec_result));
emit_final_compare_vec (s1data, s2data, result_reg,
s1addr, s2addr, orig_src1, orig_src2,
off_reg, vec_result);
emit_label (final_move_label);
emit_insn (gen_movsi (target,
gen_lowpart (SImode, result_reg)));
}
else
{ /* generate GPR code */
rtx convert_label = NULL;
rtx sub_result = gen_reg_rtx (word_mode);
bool need_6432_conversion =
expand_block_compare_gpr(bytes, base_align,
orig_src1, orig_src2,
sub_result, cond, target,
&convert_label, &final_label);
if (need_6432_conversion)
{
if (convert_label)
emit_label (convert_label);
if (TARGET_P9_MISC)
emit_insn (gen_setb_unsigned (target, cond));
else
generate_6432_conversion(target, sub_result);
}
}
if (final_label)
emit_label (final_label);
return true;
}
/* Generate page crossing check and branch code to set up for
strncmp when we don't have DI alignment.
STRNCMP_LABEL is the label to branch if there is a page crossing.
SRC_ADDR is the string address to be examined.
BYTES is the max number of bytes to compare. */
static void
expand_strncmp_align_check (rtx strncmp_label, rtx src_addr, HOST_WIDE_INT bytes)
{
rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, strncmp_label);
rtx src_pgoff = gen_reg_rtx (GET_MODE (src_addr));
do_and3 (src_pgoff, src_addr, GEN_INT (0xfff));
rtx cond = gen_reg_rtx (CCmode);
emit_move_insn (cond, gen_rtx_COMPARE (CCmode, src_pgoff,
GEN_INT (4096 - bytes)));
rtx cmp_rtx = gen_rtx_GE (VOIDmode, cond, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
lab_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, profile_probability::unlikely ());
JUMP_LABEL (j) = strncmp_label;
LABEL_NUSES (strncmp_label) += 1;
}
/* Generate the sequence of compares for strcmp/strncmp using gpr instructions.
BYTES_TO_COMPARE is the number of bytes to be compared.
BASE_ALIGN is the smaller of the alignment of the two strings.
ORIG_SRC1 is the unmodified rtx for the first string.
ORIG_SRC2 is the unmodified rtx for the second string.
TMP_REG_SRC1 is the register for loading the first string.
TMP_REG_SRC2 is the register for loading the second string.
RESULT_REG is the rtx for the result register.
EQUALITY_COMPARE_REST is a flag to indicate we need to make a cleanup call
to strcmp/strncmp if we have equality at the end of the inline comparison.
P_CLEANUP_LABEL is a pointer to rtx for a label we generate if we need code
to clean up and generate the final comparison result.
FINAL_MOVE_LABEL is rtx for a label we can branch to when we can just
set the final result. */
static void
expand_strncmp_gpr_sequence (unsigned HOST_WIDE_INT bytes_to_compare,
unsigned int base_align,
rtx orig_src1, rtx orig_src2,
rtx tmp_reg_src1, rtx tmp_reg_src2, rtx result_reg,
bool equality_compare_rest, rtx *p_cleanup_label,
rtx final_move_label)
{
unsigned int word_mode_size = GET_MODE_SIZE (word_mode);
machine_mode load_mode;
unsigned int load_mode_size;
unsigned HOST_WIDE_INT cmp_bytes = 0;
unsigned HOST_WIDE_INT offset = 0;
rtx src1_addr = force_reg (Pmode, XEXP (orig_src1, 0));
rtx src2_addr = force_reg (Pmode, XEXP (orig_src2, 0));
gcc_assert (p_cleanup_label != NULL);
rtx cleanup_label = *p_cleanup_label;
while (bytes_to_compare > 0)
{
/* GPR compare sequence:
check each 8B with: ld/ld/cmpb/cmpb/orc./bne
cleanup code at end:
cntlzd get bit of first zero/diff byte
subfic convert for rldcl use
rldcl rldcl extract diff/zero byte
subf subtract for final result
The last compare can branch around the cleanup code if the
result is zero because the strings are exactly equal. */
unsigned int align = compute_current_alignment (base_align, offset);
load_mode = select_block_compare_mode (offset, bytes_to_compare, align);
load_mode_size = GET_MODE_SIZE (load_mode);
if (bytes_to_compare >= load_mode_size)
cmp_bytes = load_mode_size;
else if (TARGET_EFFICIENT_OVERLAPPING_UNALIGNED)
{
/* Move this load back so it doesn't go past the end.
P8/P9 can do this efficiently. */
unsigned int extra_bytes = load_mode_size - bytes_to_compare;
cmp_bytes = bytes_to_compare;
if (extra_bytes < offset)
{
offset -= extra_bytes;
cmp_bytes = load_mode_size;
bytes_to_compare = cmp_bytes;
}
}
else
/* P7 and earlier can't do the overlapping load trick fast,
so this forces a non-overlapping load and a shift to get
rid of the extra bytes. */
cmp_bytes = bytes_to_compare;
rtx offset_rtx;
if (BYTES_BIG_ENDIAN || TARGET_AVOID_XFORM)
offset_rtx = GEN_INT (offset);
else
{
offset_rtx = gen_reg_rtx (Pmode);
emit_move_insn (offset_rtx, GEN_INT (offset));
}
rtx addr1 = gen_rtx_PLUS (Pmode, src1_addr, offset_rtx);
rtx addr2 = gen_rtx_PLUS (Pmode, src2_addr, offset_rtx);
do_load_for_compare_from_addr (load_mode, tmp_reg_src1, addr1, orig_src1);
do_load_for_compare_from_addr (load_mode, tmp_reg_src2, addr2, orig_src2);
/* We must always left-align the data we read, and
clear any bytes to the right that are beyond the string.
Otherwise the cmpb sequence won't produce the correct
results. However if there is only one byte left, we
can just subtract to get the final result so the shifts
and clears are not needed. */
unsigned HOST_WIDE_INT remain = bytes_to_compare - cmp_bytes;
/* Loading just a single byte is a special case. If we are
loading more than that, we have to check whether we are
looking at the entire chunk of data. If not, rotate left and
clear right so that bytes we aren't supposed to look at are
zeroed, and the first byte we are supposed to compare is
leftmost. */
if (load_mode_size != 1)
{
if (load_mode_size < word_mode_size)
{
/* Rotate left first. */
rtx sh = GEN_INT (BITS_PER_UNIT
* (word_mode_size - load_mode_size));
do_rotl3 (tmp_reg_src1, tmp_reg_src1, sh);
do_rotl3 (tmp_reg_src2, tmp_reg_src2, sh);
}
if (cmp_bytes < word_mode_size)
{
/* Now clear right. This plus the rotate can be
turned into a rldicr instruction. */
HOST_WIDE_INT mb = BITS_PER_UNIT * (word_mode_size - cmp_bytes);
rtx mask = GEN_INT (HOST_WIDE_INT_M1U << mb);
do_and3 (tmp_reg_src1, tmp_reg_src1, mask);
do_and3 (tmp_reg_src2, tmp_reg_src2, mask);
}
}
/* Cases to handle. A and B are chunks of the two strings.
1: Not end of comparison:
A != B: branch to cleanup code to compute result.
A == B: check for 0 byte, next block if not found.
2: End of the inline comparison:
A != B: branch to cleanup code to compute result.
A == B: check for 0 byte, call strcmp/strncmp
3: compared requested N bytes:
A == B: branch to result 0.
A != B: cleanup code to compute result. */
rtx dst_label;
if (remain > 0 || equality_compare_rest)
{
/* Branch to cleanup code, otherwise fall through to do
more compares. */
if (!cleanup_label)
cleanup_label = gen_label_rtx ();
dst_label = cleanup_label;
}
else
/* Branch to end and produce result of 0. */
dst_label = final_move_label;
if (load_mode_size == 1)
{
/* Special case for comparing just single byte. */
if (equality_compare_rest)
{
/* Use subf./bne to branch to final_move_label if the
byte differs, otherwise fall through to the strncmp
call. We must also check for a zero byte here as we
must not make the library call if this is the end of
the string. */
rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, final_move_label);
rtx cond = gen_reg_rtx (CCmode);
rtx diff_rtx = gen_rtx_MINUS (word_mode,
tmp_reg_src1, tmp_reg_src2);
rs6000_emit_dot_insn (result_reg, diff_rtx, 2, cond);
rtx cmp_rtx = gen_rtx_NE (VOIDmode, cond, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
lab_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, profile_probability::unlikely ());
JUMP_LABEL (j) = final_move_label;
LABEL_NUSES (final_move_label) += 1;
/* Check for zero byte here before fall through to
library call. This catches the case where the
strings are equal and end in a zero byte at this
position. */
rtx cond0 = gen_reg_rtx (CCmode);
emit_move_insn (cond0, gen_rtx_COMPARE (CCmode, tmp_reg_src1,
const0_rtx));
rtx cmp0eq_rtx = gen_rtx_EQ (VOIDmode, cond0, const0_rtx);
rtx ifelse0 = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp0eq_rtx,
lab_ref, pc_rtx);
rtx_insn *j0 = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse0));
add_reg_br_prob_note (j0, profile_probability::unlikely ());
JUMP_LABEL (j0) = final_move_label;
LABEL_NUSES (final_move_label) += 1;
}
else
{
/* This is the last byte to be compared so we can use
subf to compute the final result and branch
unconditionally to final_move_label. */
do_sub3 (result_reg, tmp_reg_src1, tmp_reg_src2);
rtx fin_ref = gen_rtx_LABEL_REF (VOIDmode, final_move_label);
rtx j = emit_jump_insn (gen_rtx_SET (pc_rtx, fin_ref));
JUMP_LABEL (j) = final_move_label;
LABEL_NUSES (final_move_label) += 1;
emit_barrier ();
}
}
else
{
rtx cmpb_zero = gen_reg_rtx (word_mode);
rtx cmpb_diff = gen_reg_rtx (word_mode);
rtx zero_reg = gen_reg_rtx (word_mode);
rtx lab_ref = gen_rtx_LABEL_REF (VOIDmode, dst_label);
rtx cond = gen_reg_rtx (CCmode);
emit_move_insn (zero_reg, GEN_INT (0));
do_cmpb3 (cmpb_diff, tmp_reg_src1, tmp_reg_src2);
do_cmpb3 (cmpb_zero, tmp_reg_src1, zero_reg);
rtx not_diff = gen_rtx_NOT (word_mode, cmpb_diff);
rtx orc_rtx = gen_rtx_IOR (word_mode, not_diff, cmpb_zero);
rs6000_emit_dot_insn (result_reg, orc_rtx, 2, cond);
rtx cmp_rtx;
if (remain == 0 && !equality_compare_rest)
cmp_rtx = gen_rtx_EQ (VOIDmode, cond, const0_rtx);
else
cmp_rtx = gen_rtx_NE (VOIDmode, cond, const0_rtx);
rtx ifelse = gen_rtx_IF_THEN_ELSE (VOIDmode, cmp_rtx,
lab_ref, pc_rtx);
rtx_insn *j = emit_jump_insn (gen_rtx_SET (pc_rtx, ifelse));
add_reg_br_prob_note (j, profile_probability::unlikely ());
JUMP_LABEL (j) = dst_label;
LABEL_NUSES (dst_label) += 1;
}
offset += cmp_bytes;
bytes_to_compare -= cmp_bytes;
}
*p_cleanup_label = cleanup_label;
return;
}
/* Generate the final sequence that identifies the differing
byte and generates the final result, taking into account
zero bytes:
cntlzd get bit of first zero/diff byte
addi convert for rldcl use
rldcl rldcl extract diff/zero byte
subf subtract for final result
STR1 is the reg rtx for data from string 1.
STR2 is the reg rtx for data from string 2.
RESULT is the reg rtx for the comparison result. */
static void
emit_final_str_compare_gpr (rtx str1, rtx str2, rtx result)
{
machine_mode m = GET_MODE (str1);
rtx rot_amt = gen_reg_rtx (m);
rtx rot1_1 = gen_reg_rtx (m);
rtx rot1_2 = gen_reg_rtx (m);
rtx rot2_1 = gen_reg_rtx (m);
rtx rot2_2 = gen_reg_rtx (m);
if (m == SImode)
{
emit_insn (gen_clzsi2 (rot_amt, result));
emit_insn (gen_addsi3 (rot_amt, rot_amt, GEN_INT (8)));
emit_insn (gen_rotlsi3 (rot1_1, str1,
gen_lowpart (SImode, rot_amt)));
emit_insn (gen_andsi3_mask (rot1_2, rot1_1, GEN_INT (0xff)));
emit_insn (gen_rotlsi3 (rot2_1, str2,
gen_lowpart (SImode, rot_amt)));
emit_insn (gen_andsi3_mask (rot2_2, rot2_1, GEN_INT (0xff)));
emit_insn (gen_subsi3 (result, rot1_2, rot2_2));
}
else if (m == DImode)
{
emit_insn (gen_clzdi2 (rot_amt, result));
emit_insn (gen_adddi3 (rot_amt, rot_amt, GEN_INT (8)));
emit_insn (gen_rotldi3 (rot1_1, str1,
gen_lowpart (SImode, rot_amt)));
emit_insn (gen_anddi3_mask (rot1_2, rot1_1, GEN_INT (0xff)));
emit_insn (gen_rotldi3 (rot2_1, str2,
gen_lowpart (SImode, rot_amt)));
emit_insn (gen_anddi3_mask (rot2_2, rot2_1, GEN_INT (0xff)));
emit_insn (gen_subdi3 (result, rot1_2, rot2_2));
}
else
gcc_unreachable ();
return;
}
/* Expand a string compare operation with length, and return
true if successful. Return false if we should let the
compiler generate normal code, probably a strncmp call.
OPERANDS[0] is the target (result).
OPERANDS[1] is the first source.
OPERANDS[2] is the second source.
If NO_LENGTH is zero, then:
OPERANDS[3] is the length.
OPERANDS[4] is the alignment in bytes.
If NO_LENGTH is nonzero, then:
OPERANDS[3] is the alignment in bytes. */
bool
expand_strn_compare (rtx operands[], int no_length)
{
rtx target = operands[0];
rtx orig_src1 = operands[1];
rtx orig_src2 = operands[2];
rtx bytes_rtx, align_rtx;
if (no_length)
{
bytes_rtx = NULL;
align_rtx = operands[3];
}
else
{
bytes_rtx = operands[3];
align_rtx = operands[4];
}
rtx src1_addr = force_reg (Pmode, XEXP (orig_src1, 0));
rtx src2_addr = force_reg (Pmode, XEXP (orig_src2, 0));
/* If we have a length, it must be constant. This simplifies things
a bit as we don't have to generate code to check if we've exceeded
the length. Later this could be expanded to handle this case. */
if (!no_length && !CONST_INT_P (bytes_rtx))
return false;
/* This must be a fixed size alignment. */
if (!CONST_INT_P (align_rtx))
return false;
unsigned int base_align = UINTVAL (align_rtx);
unsigned int align1 = MEM_ALIGN (orig_src1) / BITS_PER_UNIT;
unsigned int align2 = MEM_ALIGN (orig_src2) / BITS_PER_UNIT;
/* targetm.slow_unaligned_access -- don't do unaligned stuff. */
if (targetm.slow_unaligned_access (word_mode, align1)
|| targetm.slow_unaligned_access (word_mode, align2))
return false;
gcc_assert (GET_MODE (target) == SImode);
unsigned int required_align = 8;
unsigned HOST_WIDE_INT offset = 0;
unsigned HOST_WIDE_INT bytes; /* N from the strncmp args if available. */
unsigned HOST_WIDE_INT compare_length; /* How much to compare inline. */
if (no_length)
bytes = rs6000_string_compare_inline_limit;
else
bytes = UINTVAL (bytes_rtx);
/* Is it OK to use vec/vsx for this. TARGET_VSX means we have at
least POWER7 but we use TARGET_EFFICIENT_UNALIGNED_VSX which is
at least POWER8. That way we can rely on overlapping compares to
do the final comparison of less than 16 bytes. Also I do not
want to deal with making this work for 32 bits. In addition, we
have to make sure that we have at least P8_VECTOR (we don't allow
P9_VECTOR without P8_VECTOR). */
int use_vec = (bytes >= 16 && !TARGET_32BIT
&& TARGET_EFFICIENT_UNALIGNED_VSX && TARGET_P8_VECTOR);
if (use_vec)
required_align = 16;
machine_mode load_mode;
rtx tmp_reg_src1, tmp_reg_src2;
if (use_vec)
{
load_mode = V16QImode;
tmp_reg_src1 = gen_reg_rtx (V16QImode);
tmp_reg_src2 = gen_reg_rtx (V16QImode);
}
else
{
load_mode = select_block_compare_mode (0, bytes, base_align);
tmp_reg_src1 = gen_reg_rtx (word_mode);
tmp_reg_src2 = gen_reg_rtx (word_mode);
}
compare_length = rs6000_string_compare_inline_limit;
/* If we have equality at the end of the last compare and we have not
found the end of the string, we need to call strcmp/strncmp to
compare the remainder. */
bool equality_compare_rest = false;
if (no_length)
{
bytes = compare_length;
equality_compare_rest = true;
}
else
{
if (bytes <= compare_length)
compare_length = bytes;
else
equality_compare_rest = true;
}
rtx result_reg = gen_reg_rtx (word_mode);
rtx final_move_label = gen_label_rtx ();
rtx final_label = gen_label_rtx ();
rtx begin_compare_label = NULL;
if (base_align < required_align)
{
/* Generate code that checks distance to 4k boundary for this case. */
begin_compare_label = gen_label_rtx ();
rtx strncmp_label = gen_label_rtx ();
rtx jmp;
/* Strncmp for power8 in glibc does this:
rldicl r8,r3,0,52
cmpldi cr7,r8,4096-16
bgt cr7,L(pagecross) */
/* Make sure that the length we use for the alignment test and
the subsequent code generation are in agreement so we do not
go past the length we tested for a 4k boundary crossing. */
unsigned HOST_WIDE_INT align_test = compare_length;
if (align_test < required_align)
{
align_test = HOST_WIDE_INT_1U << ceil_log2 (align_test);
base_align = align_test;
}
else
{
align_test = ROUND_UP (align_test, required_align);
base_align = required_align;
}
if (align1 < required_align)
expand_strncmp_align_check (strncmp_label, src1_addr, align_test);
if (align2 < required_align)
expand_strncmp_align_check (strncmp_label, src2_addr, align_test);
/* Now generate the following sequence:
- branch to begin_compare
- strncmp_label
- call to strncmp
- branch to final_label
- begin_compare_label */
rtx cmp_ref = gen_rtx_LABEL_REF (VOIDmode, begin_compare_label);
jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, cmp_ref));
JUMP_LABEL (jmp) = begin_compare_label;
LABEL_NUSES (begin_compare_label) += 1;
emit_barrier ();
emit_label (strncmp_label);
if (no_length)
{
tree fun = builtin_decl_explicit (BUILT_IN_STRCMP);
emit_library_call_value (XEXP (DECL_RTL (fun), 0),
target, LCT_NORMAL, GET_MODE (target),
force_reg (Pmode, src1_addr), Pmode,
force_reg (Pmode, src2_addr), Pmode);
}
else
{
/* -m32 -mpowerpc64 results in word_mode being DImode even
though otherwise it is 32-bit. The length arg to strncmp
is a size_t which will be the same size as pointers. */
rtx len_rtx = gen_reg_rtx (Pmode);
emit_move_insn (len_rtx, gen_int_mode (bytes, Pmode));
tree fun = builtin_decl_explicit (BUILT_IN_STRNCMP);
emit_library_call_value (XEXP (DECL_RTL (fun), 0),
target, LCT_NORMAL, GET_MODE (target),
force_reg (Pmode, src1_addr), Pmode,
force_reg (Pmode, src2_addr), Pmode,
len_rtx, Pmode);
}
rtx fin_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, fin_ref));
JUMP_LABEL (jmp) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
emit_label (begin_compare_label);
}
rtx cleanup_label = NULL;
rtx s1addr = NULL, s2addr = NULL, off_reg = NULL, vec_result = NULL;
/* Generate a sequence of GPR or VEC/VSX instructions to compare out
to the length specified. */
if (use_vec)
{
s1addr = gen_reg_rtx (Pmode);
s2addr = gen_reg_rtx (Pmode);
off_reg = gen_reg_rtx (Pmode);
vec_result = gen_reg_rtx (load_mode);
emit_move_insn (result_reg, GEN_INT (0));
expand_cmp_vec_sequence (compare_length,
orig_src1, orig_src2,
s1addr, s2addr, off_reg,
tmp_reg_src1, tmp_reg_src2,
vec_result,
equality_compare_rest,
&cleanup_label, final_move_label, true);
}
else
expand_strncmp_gpr_sequence (compare_length, base_align,
orig_src1, orig_src2,
tmp_reg_src1, tmp_reg_src2,
result_reg,
equality_compare_rest,
&cleanup_label, final_move_label);
offset = compare_length;
if (equality_compare_rest)
{
/* Update pointers past what has been compared already. */
rtx src1 = force_reg (Pmode,
gen_rtx_PLUS (Pmode, src1_addr, GEN_INT (offset)));
rtx src2 = force_reg (Pmode,
gen_rtx_PLUS (Pmode, src2_addr, GEN_INT (offset)));
/* Construct call to strcmp/strncmp to compare the rest of the string. */
if (no_length)
{
tree fun = builtin_decl_explicit (BUILT_IN_STRCMP);
emit_library_call_value (XEXP (DECL_RTL (fun), 0),
target, LCT_NORMAL, GET_MODE (target),
src1, Pmode, src2, Pmode);
}
else
{
rtx len_rtx = gen_reg_rtx (Pmode);
emit_move_insn (len_rtx, gen_int_mode (bytes - compare_length, Pmode));
tree fun = builtin_decl_explicit (BUILT_IN_STRNCMP);
emit_library_call_value (XEXP (DECL_RTL (fun), 0),
target, LCT_NORMAL, GET_MODE (target),
src1, Pmode, src2, Pmode, len_rtx, Pmode);
}
rtx fin_ref = gen_rtx_LABEL_REF (VOIDmode, final_label);
rtx jmp = emit_jump_insn (gen_rtx_SET (pc_rtx, fin_ref));
JUMP_LABEL (jmp) = final_label;
LABEL_NUSES (final_label) += 1;
emit_barrier ();
}
if (cleanup_label)
emit_label (cleanup_label);
if (use_vec)
emit_final_compare_vec (tmp_reg_src1, tmp_reg_src2, result_reg,
s1addr, s2addr, orig_src1, orig_src2,
off_reg, vec_result);
else
emit_final_str_compare_gpr (tmp_reg_src1, tmp_reg_src2, result_reg);
emit_label (final_move_label);
emit_insn (gen_movsi (target,
gen_lowpart (SImode, result_reg)));
emit_label (final_label);
return true;
}
/* Generate loads and stores for a move of v4si mode using lvx/stvx.
This uses altivec_{l,st}vx__internal which use unspecs to
keep combine from changing what instruction gets used.
DEST is the destination for the data.
SRC is the source of the data for the move. */
static rtx
gen_lvx_v4si_move (rtx dest, rtx src)
{
gcc_assert (MEM_P (dest) ^ MEM_P (src));
gcc_assert (GET_MODE (dest) == V4SImode && GET_MODE (src) == V4SImode);
if (MEM_P (dest))
return gen_altivec_stvx_v4si_internal (dest, src);
else
return gen_altivec_lvx_v4si_internal (dest, src);
}
static rtx
gen_lxvl_stxvl_move (rtx dest, rtx src, int length)
{
gcc_assert (MEM_P (dest) ^ MEM_P (src));
gcc_assert (GET_MODE (dest) == V16QImode && GET_MODE (src) == V16QImode);
gcc_assert (length <= 16);
bool is_store = MEM_P (dest);
rtx addr;
/* If the address form is not a simple register, make it so. */
if (is_store)
addr = XEXP (dest, 0);
else
addr = XEXP (src, 0);
if (!REG_P (addr))
addr = force_reg (Pmode, addr);
rtx len = force_reg (DImode, gen_int_mode (length, DImode));
if (is_store)
return gen_stxvl (src, addr, len);
else
return gen_lxvl (dest, addr, len);
}
/* Expand a block move operation, and return 1 if successful. Return 0
if we should let the compiler generate normal code.
operands[0] is the destination
operands[1] is the source
operands[2] is the length
operands[3] is the alignment */
#define MAX_MOVE_REG 4
int
expand_block_move (rtx operands[], bool might_overlap)
{
rtx orig_dest = operands[0];
rtx orig_src = operands[1];
rtx bytes_rtx = operands[2];
rtx align_rtx = operands[3];
int constp = CONST_INT_P (bytes_rtx);
int align;
int bytes;
int offset;
int move_bytes;
rtx loads[MAX_MOVE_REG];
rtx stores[MAX_MOVE_REG];
int num_reg = 0;
/* If this is not a fixed size move, just call memcpy */
if (! constp)
return 0;
/* This must be a fixed size alignment */
gcc_assert (CONST_INT_P (align_rtx));
align = INTVAL (align_rtx) * BITS_PER_UNIT;
/* Anything to move? */
bytes = INTVAL (bytes_rtx);
if (bytes <= 0)
return 1;
if (bytes > rs6000_block_move_inline_limit)
return 0;
int orig_bytes = bytes;
for (offset = 0; bytes > 0; offset += move_bytes, bytes -= move_bytes)
{
union {
rtx (*mov) (rtx, rtx);
rtx (*movlen) (rtx, rtx, int);
} gen_func;
machine_mode mode = BLKmode;
rtx src, dest;
bool move_with_length = false;
/* Use OOmode for paired vsx load/store. Use V2DI for single
unaligned vsx load/store, for consistency with what other
expansions (compare) already do, and so we can use lxvd2x on
p8. Order is VSX pair unaligned, VSX unaligned, Altivec, VSX
with length < 16 (if allowed), then gpr load/store. */
if (TARGET_MMA && TARGET_BLOCK_OPS_UNALIGNED_VSX
&& TARGET_BLOCK_OPS_VECTOR_PAIR
&& bytes >= 32
&& (align >= 256 || !STRICT_ALIGNMENT))
{
move_bytes = 32;
mode = OOmode;
gen_func.mov = gen_movoo;
}
else if (TARGET_POWERPC64 && TARGET_BLOCK_OPS_UNALIGNED_VSX
&& VECTOR_MEM_VSX_P (V2DImode)
&& bytes >= 16 && (align >= 128 || !STRICT_ALIGNMENT))
{
move_bytes = 16;
mode = V2DImode;
gen_func.mov = gen_vsx_movv2di_64bit;
}
else if (TARGET_BLOCK_OPS_UNALIGNED_VSX
&& TARGET_POWER10 && bytes < 16
&& orig_bytes > 16
&& !(bytes == 1 || bytes == 2
|| bytes == 4 || bytes == 8)
&& (align >= 128 || !STRICT_ALIGNMENT))
{
/* Only use lxvl/stxvl if it could replace multiple ordinary
loads+stores. Also don't use it unless we likely already
did one vsx copy so we aren't mixing gpr and vsx. */
move_bytes = bytes;
mode = V16QImode;
gen_func.movlen = gen_lxvl_stxvl_move;
move_with_length = true;
}
else if (TARGET_ALTIVEC && bytes >= 16 && align >= 128)
{
move_bytes = 16;
mode = V4SImode;
gen_func.mov = gen_lvx_v4si_move;
}
else if (bytes >= 8 && TARGET_POWERPC64
&& (align >= 64 || !STRICT_ALIGNMENT))
{
move_bytes = 8;
mode = DImode;
gen_func.mov = gen_movdi;
if (offset == 0 && align < 64)
{
rtx addr;
/* If the address form is reg+offset with offset not a
multiple of four, reload into reg indirect form here
rather than waiting for reload. This way we get one
reload, not one per load and/or store. */
addr = XEXP (orig_dest, 0);
if ((GET_CODE (addr) == PLUS || GET_CODE (addr) == LO_SUM)
&& CONST_INT_P (XEXP (addr, 1))
&& (INTVAL (XEXP (addr, 1)) & 3) != 0)
{
addr = copy_addr_to_reg (addr);
orig_dest = replace_equiv_address (orig_dest, addr);
}
addr = XEXP (orig_src, 0);
if ((GET_CODE (addr) == PLUS || GET_CODE (addr) == LO_SUM)
&& CONST_INT_P (XEXP (addr, 1))
&& (INTVAL (XEXP (addr, 1)) & 3) != 0)
{
addr = copy_addr_to_reg (addr);
orig_src = replace_equiv_address (orig_src, addr);
}
}
}
else if (bytes >= 4 && (align >= 32 || !STRICT_ALIGNMENT))
{ /* move 4 bytes */
move_bytes = 4;
mode = SImode;
gen_func.mov = gen_movsi;
}
else if (bytes >= 2 && (align >= 16 || !STRICT_ALIGNMENT))
{ /* move 2 bytes */
move_bytes = 2;
mode = HImode;
gen_func.mov = gen_movhi;
}
else /* move 1 byte at a time */
{
move_bytes = 1;
mode = QImode;
gen_func.mov = gen_movqi;
}
/* If we can't succeed in doing the move in one pass, we can't
do it in the might_overlap case. Bail out and return
failure. We test num_reg + 1 >= MAX_MOVE_REG here to check
the same condition as the test of num_reg >= MAX_MOVE_REG
that is done below after the increment of num_reg. */
if (might_overlap && num_reg + 1 >= MAX_MOVE_REG
&& bytes > move_bytes)
return 0;
/* Mode is always set to something other than BLKmode by one of the
cases of the if statement above. */
gcc_assert (mode != BLKmode);
src = adjust_address (orig_src, mode, offset);
dest = adjust_address (orig_dest, mode, offset);
rtx tmp_reg = gen_reg_rtx (mode);
if (move_with_length)
{
loads[num_reg] = (*gen_func.movlen) (tmp_reg, src, move_bytes);
stores[num_reg++] = (*gen_func.movlen) (dest, tmp_reg, move_bytes);
}
else
{
loads[num_reg] = (*gen_func.mov) (tmp_reg, src);
stores[num_reg++] = (*gen_func.mov) (dest, tmp_reg);
}
/* Emit loads and stores saved up. */
if (num_reg >= MAX_MOVE_REG || bytes == move_bytes)
{
int i;
for (i = 0; i < num_reg; i++)
emit_insn (loads[i]);
for (i = 0; i < num_reg; i++)
emit_insn (stores[i]);
num_reg = 0;
}
}
return 1;
}