diff --git a/etc/launchd.d/.gitignore b/etc/launchd.d/.gitignore new file mode 100644 index 000000000000..f11cc9cf5e4f --- /dev/null +++ b/etc/launchd.d/.gitignore @@ -0,0 +1 @@ +zpool-import-all.sh diff --git a/etc/launchd.d/Makefile.am b/etc/launchd.d/Makefile.am new file mode 100644 index 000000000000..064602465dc9 --- /dev/null +++ b/etc/launchd.d/Makefile.am @@ -0,0 +1,19 @@ + +noinst_launch_d_DATA = \ + %D%/zpool-import-all.sh.in +noinst_launch_ddir = \ + $(launchdscriptdir)/ + +do_subst = -$(SED) -e 's,@bindir\@,$(bindir),g' \ + -e 's,@runstatedir\@,$(runstatedir),g' \ + -e 's,@sbindir\@,$(sbindir),g' \ + -e 's,@sysconfdir\@,$(sysconfdir),g' \ + -e 's,@launchdscriptdir\@,$(launchdscriptdir),g' + +CLEANFILES += %D%/zpool-import-all.sh + +INSTALL_DATA_HOOKS += install-zpool-import + +install-zpool-import: %D%/zpool-import-all.sh.in + $(do_subst) < %D%/zpool-import-all.sh.in > $(DESTDIR)/$(launchdscriptdir)/zpool-import-all.sh + chmod +x $(DESTDIR)/$(launchdscriptdir)/zpool-import-all.sh diff --git a/etc/launchd.d/zpool-import-all.sh.in b/etc/launchd.d/zpool-import-all.sh.in new file mode 100755 index 000000000000..7df5eba0ca64 --- /dev/null +++ b/etc/launchd.d/zpool-import-all.sh.in @@ -0,0 +1,60 @@ +#!/bin/bash + +echo "+zpool-import-all.sh" +date +export ZPOOL_IMPORT_ALL_COOKIE=/var/run/org.openzfsonosx.zpool-import-all.didRun +export INVARIANT_DISKS_IDLE_FILE=/var/run/disk/invariant.idle +export TIMEOUT_SECONDS=60 +export MAXIMUM_SLEEP_ITERATIONS=$((${TIMEOUT_SECONDS} * 10)) + +/usr/bin/time /usr/sbin/system_profiler SPParallelATADataType SPCardReaderDataType SPFibreChannelDataType SPFireWireDataType SPHardwareRAIDDataType SPNetworkDataType SPPCIDataType SPParallelSCSIDataType SPSASDataType SPSerialATADataType SPStorageDataType SPThunderboltDataType SPUSBDataType SPNetworkVolumeDataType 1>/dev/null 2>&1 + +/bin/sync + +echo "Waiting up to ${TIMEOUT_SECONDS} seconds for the InvariantDisks idle file ${INVARIANT_DISKS_IDLE_FILE} to exist" + +i=0 +while [ "${i}" -lt "${MAXIMUM_SLEEP_ITERATIONS}" -a ! -e "${INVARIANT_DISKS_IDLE_FILE}" ] +do + i=$((i+1)) + sleep .1 +done + +if [ -e "${INVARIANT_DISKS_IDLE_FILE}" ] +then + echo "Found ${INVARIANT_DISKS_IDLE_FILE} after ${i} iterations of sleeping 0.1 seconds" +else + echo "File ${INVARIANT_DISKS_IDLE_FILE} not found within ${TIMEOUT_SECONDS} seconds" +fi +date + +if [ -f "@sysconfdir@/zfs/zsysctl.conf" ]; then + @sbindir@/zsysctl -f @sysconfdir@/zfs/zsysctl.conf +fi + +# check to see if we have been instructed not to try to import at all +if [ -f /etc/zfs/noautoimport ]; +then + echo "/etc/zfs/noautoimport exits, exiting"; + exit 0; +fi + +date +echo "-zpool-import-all.sh" + +sleep 10 +echo "Loading and starting org.openzfsonosx.zpool-import" +date + +/bin/launchctl load /Library/LaunchDaemons/org.openzfsonosx.zpool-import.plist +/bin/launchctl kickstart -kp system/org.openzfsonosx.zpool-import + +echo Status: $? + +echo "Touching the file ${ZPOOL_IMPORT_ALL_COOKIE}" +touch "${ZPOOL_IMPORT_ALL_COOKIE}" + +date +echo "-zpool-import-all.sh" + +exit 0 diff --git a/etc/launchd/Makefile.am b/etc/launchd/Makefile.am new file mode 100644 index 000000000000..70047aa02e36 --- /dev/null +++ b/etc/launchd/Makefile.am @@ -0,0 +1,4 @@ +launchddaemondir = /Library/LaunchDaemons +launchdscriptdir = ${libexecdir}/zfs/launchd.d + +include $(srcdir)/%D%/daemons/Makefile.am diff --git a/etc/launchd/daemons/.gitignore b/etc/launchd/daemons/.gitignore new file mode 100644 index 000000000000..8b43b0f2a1fd --- /dev/null +++ b/etc/launchd/daemons/.gitignore @@ -0,0 +1,5 @@ +org.openzfsonosx.zconfigd.plist +org.openzfsonosx.zed.plist +org.openzfsonosx.zpool-import.plist +org.openzfsonosx.zpool-import-all.plist +org.openzfsonosx.InvariantDisks.plist diff --git a/etc/launchd/daemons/Makefile.am b/etc/launchd/daemons/Makefile.am new file mode 100644 index 000000000000..5f16a4c1fce5 --- /dev/null +++ b/etc/launchd/daemons/Makefile.am @@ -0,0 +1,33 @@ + +CLEANFILES += $(launchddaemon_DATA) + +launchddaemon_DATA = \ + $(top_srcdir)/etc/launchd/daemons/org.openzfsonosx.zconfigd.plist \ + $(top_srcdir)/etc/launchd/daemons/org.openzfsonosx.zed.plist \ + $(top_srcdir)/etc/launchd/daemons/org.openzfsonosx.zpool-import.plist \ + $(top_srcdir)/etc/launchd/daemons/org.openzfsonosx.zpool-import-all.plist \ + $(top_srcdir)/etc/launchd/daemons/org.openzfsonosx.InvariantDisks.plist + +dist_launchddaemon_DATA = \ + %D%/org.openzfsonosx.zconfigd.plist.in \ + %D%/org.openzfsonosx.zed.plist.in \ + %D%/org.openzfsonosx.zpool-import.plist.in \ + %D%/org.openzfsonosx.zpool-import-all.plist.in \ + %D%/org.openzfsonosx.InvariantDisks.plist.in + +%D%/org.openzfsonosx.zconfigd.plist: %D%/org.openzfsonosx.zconfigd.plist.in +%D%/org.openzfsonosx.zed.plist: %D%/org.openzfsonosx.zed.plist.in +%D%/org.openzfsonosx.zpool-import.plist: %D%/org.openzfsonosx.zpool-import.plist.in +%D%/org.openzfsonosx.zpool-import-all.plist: %D%/org.openzfsonosx.zpool-import-all.plist.in +%D%/org.openzfsonosx.InvariantDisks.plist: %D%/org.openzfsonosx.InvariantDisks.plist.in + +$(launchddaemon_DATA): + -$(SED) -e 's,@bindir\@,$(bindir),g' \ + -e 's,@runstatedir\@,$(runstatedir),g' \ + -e 's,@sbindir\@,$(sbindir),g' \ + -e 's,@sysconfdir\@,$(sysconfdir),g' \ + -e 's,@launchddaemondir\@,$(launchddaemondir),g' \ + -e 's,@launchdscriptdir\@,$(launchdscriptdir),g' \ + '$@.in' >'$@' + + diff --git a/etc/launchd/daemons/org.openzfsonosx.InvariantDisks.plist.in b/etc/launchd/daemons/org.openzfsonosx.InvariantDisks.plist.in new file mode 100644 index 000000000000..4e49548aedb7 --- /dev/null +++ b/etc/launchd/daemons/org.openzfsonosx.InvariantDisks.plist.in @@ -0,0 +1,16 @@ + + + + + Label + org.openzfsonosx.InvariantDisks + ProgramArguments + + @sbindir@/InvariantDisks + + RunAtLoad + + KeepAlive + + + diff --git a/etc/launchd/daemons/org.openzfsonosx.zconfigd.plist.in b/etc/launchd/daemons/org.openzfsonosx.zconfigd.plist.in new file mode 100644 index 000000000000..51154e84661a --- /dev/null +++ b/etc/launchd/daemons/org.openzfsonosx.zconfigd.plist.in @@ -0,0 +1,20 @@ + + + + + Label + org.openzfsonosx.zconfigd + ProgramArguments + + @sbindir@/zconfigd + + RunAtLoad + + KeepAlive + + StandardErrorPath + /private/var/log/org.openzfsonosx.zconfigd.err + StandardOutPath + /private/var/log/org.openzfsonosx.zconfigd.log + + diff --git a/etc/launchd/daemons/org.openzfsonosx.zed.plist.in b/etc/launchd/daemons/org.openzfsonosx.zed.plist.in new file mode 100644 index 000000000000..1f62d4e2a9d2 --- /dev/null +++ b/etc/launchd/daemons/org.openzfsonosx.zed.plist.in @@ -0,0 +1,21 @@ + + + + + Label + org.openzfsonosx.zed + ProgramArguments + + @sbindir@/zed + -vfF + + RunAtLoad + + KeepAlive + + StandardErrorPath + /private/var/log/org.openzfsonosx.zed.err + StandardOutPath + /private/var/log/org.openzfsonosx.zed.log + + diff --git a/etc/launchd/daemons/org.openzfsonosx.zpool-import-all.plist.in b/etc/launchd/daemons/org.openzfsonosx.zpool-import-all.plist.in new file mode 100644 index 000000000000..3687eb418e33 --- /dev/null +++ b/etc/launchd/daemons/org.openzfsonosx.zpool-import-all.plist.in @@ -0,0 +1,18 @@ + + + + + Label + org.openzfsonosx.zpool-import-all + ProgramArguments + + @launchdscriptdir@/zpool-import-all.sh + + RunAtLoad + + StandardErrorPath + /private/var/log/org.openzfsonosx.zpool-import-all.err + StandardOutPath + /private/var/log/org.openzfsonosx.zpool-import-all.log + + diff --git a/etc/launchd/daemons/org.openzfsonosx.zpool-import.plist.in b/etc/launchd/daemons/org.openzfsonosx.zpool-import.plist.in new file mode 100644 index 000000000000..ee60285ba19e --- /dev/null +++ b/etc/launchd/daemons/org.openzfsonosx.zpool-import.plist.in @@ -0,0 +1,24 @@ + + + + + Label + org.openzfsonosx.zpool-import + ProgramArguments + + @sbindir@/zpool + import + -a + -d + /var/run/disk/by-id + + RunAtLoad + + LaunchOnlyOnce + + StandardErrorPath + /private/var/log/org.openzfsonosx.zpool-import-all.log + StandardOutPath + /private/var/log/org.openzfsonosx.zpool-import-all.log + + diff --git a/etc/paths.d/.gitignore b/etc/paths.d/.gitignore new file mode 100644 index 000000000000..f916e9fbee04 --- /dev/null +++ b/etc/paths.d/.gitignore @@ -0,0 +1 @@ +zfs-path diff --git a/etc/paths.d/Makefile.am b/etc/paths.d/Makefile.am new file mode 100644 index 000000000000..8b7f5998b926 --- /dev/null +++ b/etc/paths.d/Makefile.am @@ -0,0 +1,20 @@ +pathsddir = $(sysconfdir)/paths.d +pathsd_DATA = %D%/zfs-path + +paths_ddir = $(sysconfdir)/paths.d/ +dist_paths_d_DATA = \ + %D%/zfs-path + +zfs-path: %D%/zfs-path.in + +CLEANFILES += $(pathsd_DATA) +CLEANFILES += %D%/zfs-path +INSTALL_DATA_HOOKS += pathsd-data-hook +pathsd-data-hook: %D%/zfs-path.in + -$(SED) -e 's,@bindir\@,$(bindir),g' \ + -e 's,@runstatedir\@,$(runstatedir),g' \ + -e 's,@sbindir\@,$(sbindir),g' \ + -e 's,@sysconfdir\@,$(sysconfdir),g' \ + -e 's,@launchddaemondir\@,$(launchddaemondir),g' \ + -e 's,@launchdscriptdir\@,$(launchdscriptdir),g' \ + %D%/zfs-path.in > $(DESTDIR)/$(sysconfdir)/paths.d/zfs-path diff --git a/etc/paths.d/zfs-path.in b/etc/paths.d/zfs-path.in new file mode 100644 index 000000000000..5532f40754b3 --- /dev/null +++ b/etc/paths.d/zfs-path.in @@ -0,0 +1,2 @@ +@bindir@ +@sbindir@ diff --git a/lib/libspl/asm-x86_64/atomic.S b/lib/libspl/asm-x86_64/atomic.S new file mode 100644 index 000000000000..bb7dff12cc8e --- /dev/null +++ b/lib/libspl/asm-x86_64/atomic.S @@ -0,0 +1,693 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or http://www.opensolaris.org/os/licensing. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ +/* + * Copyright 2007 Sun Microsystems, Inc. All rights reserved. + * Use is subject to license terms. + */ + + .ident "%Z%%M% %I% %E% SMI" + + .file "%M%" + +#define _ASM +#ifdef __linux__ +#include +#elif __FreeBSD__ +#include +#define SET_SIZE(x) +#elif __APPLE__ +#include +#endif + ENTRY(atomic_inc_8) + ALTENTRY(atomic_inc_uchar) + lock + incb (%rdi) + ret + SET_SIZE(atomic_inc_uchar) + SET_SIZE(atomic_inc_8) + + ENTRY(atomic_inc_16) + ALTENTRY(atomic_inc_ushort) + lock + incw (%rdi) + ret + SET_SIZE(atomic_inc_ushort) + SET_SIZE(atomic_inc_16) + + ENTRY(atomic_inc_32) + ALTENTRY(atomic_inc_uint) + lock + incl (%rdi) + ret + SET_SIZE(atomic_inc_uint) + SET_SIZE(atomic_inc_32) + + ENTRY(atomic_inc_64) + ALTENTRY(atomic_inc_ulong) + lock + incq (%rdi) + ret + SET_SIZE(atomic_inc_ulong) + SET_SIZE(atomic_inc_64) + + ENTRY(atomic_inc_8_nv) + ALTENTRY(atomic_inc_uchar_nv) + movb (%rdi), %al +1: + leaq 1(%rax), %rcx + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_inc_uchar_nv) + SET_SIZE(atomic_inc_8_nv) + + ENTRY(atomic_inc_16_nv) + ALTENTRY(atomic_inc_ushort_nv) + movw (%rdi), %ax +1: + leaq 1(%rax), %rcx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_inc_ushort_nv) + SET_SIZE(atomic_inc_16_nv) + + ENTRY(atomic_inc_32_nv) + ALTENTRY(atomic_inc_uint_nv) + movl (%rdi), %eax +1: + leaq 1(%rax), %rcx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_inc_uint_nv) + SET_SIZE(atomic_inc_32_nv) + + ENTRY(atomic_inc_64_nv) + ALTENTRY(atomic_inc_ulong_nv) + movq (%rdi), %rax +1: + leaq 1(%rax), %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_inc_ulong_nv) + SET_SIZE(atomic_inc_64_nv) + + ENTRY(atomic_dec_8) + ALTENTRY(atomic_dec_uchar) + lock + decb (%rdi) + ret + SET_SIZE(atomic_dec_uchar) + SET_SIZE(atomic_dec_8) + + ENTRY(atomic_dec_16) + ALTENTRY(atomic_dec_ushort) + lock + decw (%rdi) + ret + SET_SIZE(atomic_dec_ushort) + SET_SIZE(atomic_dec_16) + + ENTRY(atomic_dec_32) + ALTENTRY(atomic_dec_uint) + lock + decl (%rdi) + ret + SET_SIZE(atomic_dec_uint) + SET_SIZE(atomic_dec_32) + + ENTRY(atomic_dec_64) + ALTENTRY(atomic_dec_ulong) + lock + decq (%rdi) + ret + SET_SIZE(atomic_dec_ulong) + SET_SIZE(atomic_dec_64) + + ENTRY(atomic_dec_8_nv) + ALTENTRY(atomic_dec_uchar_nv) + movb (%rdi), %al +1: + leaq -1(%rax), %rcx + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_dec_uchar_nv) + SET_SIZE(atomic_dec_8_nv) + + ENTRY(atomic_dec_16_nv) + ALTENTRY(atomic_dec_ushort_nv) + movw (%rdi), %ax +1: + leaq -1(%rax), %rcx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_dec_ushort_nv) + SET_SIZE(atomic_dec_16_nv) + + ENTRY(atomic_dec_32_nv) + ALTENTRY(atomic_dec_uint_nv) + movl (%rdi), %eax +1: + leaq -1(%rax), %rcx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_dec_uint_nv) + SET_SIZE(atomic_dec_32_nv) + + ENTRY(atomic_dec_64_nv) + ALTENTRY(atomic_dec_ulong_nv) + movq (%rdi), %rax +1: + leaq -1(%rax), %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_dec_ulong_nv) + SET_SIZE(atomic_dec_64_nv) + + ENTRY(atomic_add_8) + ALTENTRY(atomic_add_char) + lock + addb %sil, (%rdi) + ret + SET_SIZE(atomic_add_char) + SET_SIZE(atomic_add_8) + + ENTRY(atomic_add_16) + ALTENTRY(atomic_add_short) + lock + addw %si, (%rdi) + ret + SET_SIZE(atomic_add_short) + SET_SIZE(atomic_add_16) + + ENTRY(atomic_add_32) + ALTENTRY(atomic_add_int) + lock + addl %esi, (%rdi) + ret + SET_SIZE(atomic_add_int) + SET_SIZE(atomic_add_32) + + ENTRY(atomic_add_64) + ALTENTRY(atomic_add_ptr) + ALTENTRY(atomic_add_long) + lock + addq %rsi, (%rdi) + ret + SET_SIZE(atomic_add_long) + SET_SIZE(atomic_add_ptr) + SET_SIZE(atomic_add_64) + + ENTRY(atomic_sub_8) + ALTENTRY(atomic_sub_char) + lock + subb %sil, (%rdi) + ret + SET_SIZE(atomic_sub_char) + SET_SIZE(atomic_sub_8) + + ENTRY(atomic_sub_16) + ALTENTRY(atomic_sub_short) + lock + subw %si, (%rdi) + ret + SET_SIZE(atomic_sub_short) + SET_SIZE(atomic_sub_16) + + ENTRY(atomic_sub_32) + ALTENTRY(atomic_sub_int) + lock + subl %esi, (%rdi) + ret + SET_SIZE(atomic_sub_int) + SET_SIZE(atomic_sub_32) + + ENTRY(atomic_sub_64) + ALTENTRY(atomic_sub_ptr) + ALTENTRY(atomic_sub_long) + lock + subq %rsi, (%rdi) + ret + SET_SIZE(atomic_sub_long) + SET_SIZE(atomic_sub_ptr) + SET_SIZE(atomic_sub_64) + + ENTRY(atomic_or_8) + ALTENTRY(atomic_or_uchar) + lock + orb %sil, (%rdi) + ret + SET_SIZE(atomic_or_uchar) + SET_SIZE(atomic_or_8) + + ENTRY(atomic_or_16) + ALTENTRY(atomic_or_ushort) + lock + orw %si, (%rdi) + ret + SET_SIZE(atomic_or_ushort) + SET_SIZE(atomic_or_16) + + ENTRY(atomic_or_32) + ALTENTRY(atomic_or_uint) + lock + orl %esi, (%rdi) + ret + SET_SIZE(atomic_or_uint) + SET_SIZE(atomic_or_32) + + ENTRY(atomic_or_64) + ALTENTRY(atomic_or_ulong) + lock + orq %rsi, (%rdi) + ret + SET_SIZE(atomic_or_ulong) + SET_SIZE(atomic_or_64) + + ENTRY(atomic_and_8) + ALTENTRY(atomic_and_uchar) + lock + andb %sil, (%rdi) + ret + SET_SIZE(atomic_and_uchar) + SET_SIZE(atomic_and_8) + + ENTRY(atomic_and_16) + ALTENTRY(atomic_and_ushort) + lock + andw %si, (%rdi) + ret + SET_SIZE(atomic_and_ushort) + SET_SIZE(atomic_and_16) + + ENTRY(atomic_and_32) + ALTENTRY(atomic_and_uint) + lock + andl %esi, (%rdi) + ret + SET_SIZE(atomic_and_uint) + SET_SIZE(atomic_and_32) + + ENTRY(atomic_and_64) + ALTENTRY(atomic_and_ulong) + lock + andq %rsi, (%rdi) + ret + SET_SIZE(atomic_and_ulong) + SET_SIZE(atomic_and_64) + + ENTRY(atomic_add_8_nv) + ALTENTRY(atomic_add_char_nv) + movb (%rdi), %al +1: + movb %sil, %cl + addb %al, %cl + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_add_char_nv) + SET_SIZE(atomic_add_8_nv) + + ENTRY(atomic_add_16_nv) + ALTENTRY(atomic_add_short_nv) + movw (%rdi), %ax +1: + movw %si, %cx + addw %ax, %cx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_add_short_nv) + SET_SIZE(atomic_add_16_nv) + + ENTRY(atomic_add_32_nv) + ALTENTRY(atomic_add_int_nv) + movl (%rdi), %eax +1: + movl %esi, %ecx + addl %eax, %ecx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_add_int_nv) + SET_SIZE(atomic_add_32_nv) + + ENTRY(atomic_add_64_nv) + ALTENTRY(atomic_add_ptr_nv) + ALTENTRY(atomic_add_long_nv) + movq (%rdi), %rax +1: + movq %rsi, %rcx + addq %rax, %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_add_long_nv) + SET_SIZE(atomic_add_ptr_nv) + SET_SIZE(atomic_add_64_nv) + + ENTRY(atomic_sub_8_nv) + ALTENTRY(atomic_sub_char_nv) + movb (%rdi), %al +1: + movb %sil, %cl + subb %al, %cl + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_sub_char_nv) + SET_SIZE(atomic_sub_8_nv) + + ENTRY(atomic_sub_16_nv) + ALTENTRY(atomic_sub_short_nv) + movw (%rdi), %ax +1: + movw %si, %cx + subw %ax, %cx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_sub_short_nv) + SET_SIZE(atomic_sub_16_nv) + + ENTRY(atomic_sub_32_nv) + ALTENTRY(atomic_sub_int_nv) + movl (%rdi), %eax +1: + movl %esi, %ecx + subl %eax, %ecx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_sub_int_nv) + SET_SIZE(atomic_sub_32_nv) + + ENTRY(atomic_sub_64_nv) + ALTENTRY(atomic_sub_ptr_nv) + ALTENTRY(atomic_sub_long_nv) + movq (%rdi), %rax +1: + movq %rsi, %rcx + subq %rax, %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_sub_long_nv) + SET_SIZE(atomic_sub_ptr_nv) + SET_SIZE(atomic_sub_64_nv) + + ENTRY(atomic_and_8_nv) + ALTENTRY(atomic_and_uchar_nv) + movb (%rdi), %al +1: + movb %sil, %cl + andb %al, %cl + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_and_uchar_nv) + SET_SIZE(atomic_and_8_nv) + + ENTRY(atomic_and_16_nv) + ALTENTRY(atomic_and_ushort_nv) + movw (%rdi), %ax +1: + movw %si, %cx + andw %ax, %cx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_and_ushort_nv) + SET_SIZE(atomic_and_16_nv) + + ENTRY(atomic_and_32_nv) + ALTENTRY(atomic_and_uint_nv) + movl (%rdi), %eax +1: + movl %esi, %ecx + andl %eax, %ecx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_and_uint_nv) + SET_SIZE(atomic_and_32_nv) + + ENTRY(atomic_and_64_nv) + ALTENTRY(atomic_and_ulong_nv) + movq (%rdi), %rax +1: + movq %rsi, %rcx + andq %rax, %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_and_ulong_nv) + SET_SIZE(atomic_and_64_nv) + + ENTRY(atomic_or_8_nv) + ALTENTRY(atomic_or_uchar_nv) + movb (%rdi), %al +1: + movb %sil, %cl + orb %al, %cl + lock + cmpxchgb %cl, (%rdi) + jne 1b + movzbl %cl, %eax + ret + SET_SIZE(atomic_and_uchar_nv) + SET_SIZE(atomic_and_8_nv) + + ENTRY(atomic_or_16_nv) + ALTENTRY(atomic_or_ushort_nv) + movw (%rdi), %ax +1: + movw %si, %cx + orw %ax, %cx + lock + cmpxchgw %cx, (%rdi) + jne 1b + movzwl %cx, %eax + ret + SET_SIZE(atomic_or_ushort_nv) + SET_SIZE(atomic_or_16_nv) + + ENTRY(atomic_or_32_nv) + ALTENTRY(atomic_or_uint_nv) + movl (%rdi), %eax +1: + movl %esi, %ecx + orl %eax, %ecx + lock + cmpxchgl %ecx, (%rdi) + jne 1b + movl %ecx, %eax + ret + SET_SIZE(atomic_or_uint_nv) + SET_SIZE(atomic_or_32_nv) + + ENTRY(atomic_or_64_nv) + ALTENTRY(atomic_or_ulong_nv) + movq (%rdi), %rax +1: + movq %rsi, %rcx + orq %rax, %rcx + lock + cmpxchgq %rcx, (%rdi) + jne 1b + movq %rcx, %rax + ret + SET_SIZE(atomic_or_ulong_nv) + SET_SIZE(atomic_or_64_nv) + + ENTRY(atomic_cas_8) + ALTENTRY(atomic_cas_uchar) + movzbl %sil, %eax + lock + cmpxchgb %dl, (%rdi) + ret + SET_SIZE(atomic_cas_uchar) + SET_SIZE(atomic_cas_8) + + ENTRY(atomic_cas_16) + ALTENTRY(atomic_cas_ushort) + movzwl %si, %eax + lock + cmpxchgw %dx, (%rdi) + ret + SET_SIZE(atomic_cas_ushort) + SET_SIZE(atomic_cas_16) + + ENTRY(atomic_cas_32) + ALTENTRY(atomic_cas_uint) + movl %esi, %eax + lock + cmpxchgl %edx, (%rdi) + ret + SET_SIZE(atomic_cas_uint) + SET_SIZE(atomic_cas_32) + + ENTRY(atomic_cas_64) + ALTENTRY(atomic_cas_ulong) + ALTENTRY(atomic_cas_ptr) + movq %rsi, %rax + lock + cmpxchgq %rdx, (%rdi) + ret + SET_SIZE(atomic_cas_ptr) + SET_SIZE(atomic_cas_ulong) + SET_SIZE(atomic_cas_64) + + ENTRY(atomic_swap_8) + ALTENTRY(atomic_swap_uchar) + movzbl %sil, %eax + lock + xchgb %al, (%rdi) + ret + SET_SIZE(atomic_swap_uchar) + SET_SIZE(atomic_swap_8) + + ENTRY(atomic_swap_16) + ALTENTRY(atomic_swap_ushort) + movzwl %si, %eax + lock + xchgw %ax, (%rdi) + ret + SET_SIZE(atomic_swap_ushort) + SET_SIZE(atomic_swap_16) + + ENTRY(atomic_swap_32) + ALTENTRY(atomic_swap_uint) + movl %esi, %eax + lock + xchgl %eax, (%rdi) + ret + SET_SIZE(atomic_swap_uint) + SET_SIZE(atomic_swap_32) + + ENTRY(atomic_swap_64) + ALTENTRY(atomic_swap_ulong) + ALTENTRY(atomic_swap_ptr) + movq %rsi, %rax + lock + xchgq %rax, (%rdi) + ret + SET_SIZE(atomic_swap_ptr) + SET_SIZE(atomic_swap_ulong) + SET_SIZE(atomic_swap_64) + + ENTRY(atomic_set_long_excl) + xorl %eax, %eax + lock + btsq %rsi, (%rdi) + jnc 1f + decl %eax +1: + ret + SET_SIZE(atomic_set_long_excl) + + ENTRY(atomic_clear_long_excl) + xorl %eax, %eax + lock + btrq %rsi, (%rdi) + jc 1f + decl %eax +1: + ret + SET_SIZE(atomic_clear_long_excl) + + /* + * NOTE: membar_enter, and membar_exit are identical routines. + * We define them separately, instead of using an ALTENTRY + * definitions to alias them together, so that DTrace and + * debuggers will see a unique address for them, allowing + * more accurate tracing. + */ + + ENTRY(membar_enter) + mfence + ret + SET_SIZE(membar_enter) + + ENTRY(membar_exit) + mfence + ret + SET_SIZE(membar_exit) + + ENTRY(membar_producer) + sfence + ret + SET_SIZE(membar_producer) + + ENTRY(membar_consumer) + lfence + ret + SET_SIZE(membar_consumer) + +#ifdef __ELF__ +.section .note.GNU-stack,"",%progbits +#endif