Skip to content

Commit

Permalink
update longtail to 0.4.0 (#246)
Browse files Browse the repository at this point in the history
- **NEW** added builds for Arm64 flavours
  - meow hash option is not supported on Arm64 and will return 0 if called
  - blake2 hash option is not supported on Arm64 and will return 0 if called
  - `macos-arm64.zip` and `longtail-macos-arm64` artifacts are produced when creating a release
- **UPDATED** Updated to longtail 0.4.0
  • Loading branch information
DanEngelbrecht authored Sep 8, 2023
1 parent 4469fb5 commit 6076be7
Show file tree
Hide file tree
Showing 11 changed files with 165 additions and 21 deletions.
64 changes: 45 additions & 19 deletions .github/workflows/create-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,26 @@ jobs:

strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
arch: [amd64, arm64]
include:
- os: ubuntu-20.04
target: "./cmd/longtail/longtail"
platform: linux
tag: "${GITHUB_REF_NAME}"
- os: macos-latest
target: "./cmd/longtail/longtail"
platform: macos
tag: "${GITHUB_REF_NAME}"
- os: windows-latest
target: "./cmd/longtail/longtail.exe"
platform: win32
tag: "${env:GITHUB_REF_NAME}"
- os: ubuntu-20.04
target: "./cmd/longtail/longtail"
platform: linux
tag: "${GITHUB_REF_NAME}"
- os: macos-latest
target: "./cmd/longtail/longtail"
platform: macos
tag: "${GITHUB_REF_NAME}"
- os: windows-latest
target: "./cmd/longtail/longtail.exe"
platform: win32
tag: "${env:GITHUB_REF_NAME}"
exclude:
- os: ubuntu-20.04
arch: arm64
- os: windows-latest
arch: arm64

runs-on: ${{matrix.os}}

Expand All @@ -39,10 +46,14 @@ jobs:

- name: Build
run: |
echo "GOARCH: " $GOARCH
pushd ./cmd/longtail
echo matrix.tag ${{ matrix.tag }}
go build -ldflags="-s -w -X 'github.com/DanEngelbrecht/golongtail/commands.BuildVersion=${{ matrix.tag }}'" .
popd
env:
GOARCH: ${{matrix.arch}}
CGO_ENABLED: 1

- name: build dist
run: |
Expand All @@ -52,7 +63,7 @@ jobs:
- name: Upload artifacts
uses: actions/upload-artifact@master
with:
name: dist-${{matrix.platform}}-x64
name: dist-${{matrix.platform}}-${{matrix.arch}}
path: dist

create-release:
Expand All @@ -68,42 +79,55 @@ jobs:
- name: Download Linux artifacts
uses: actions/download-artifact@v1
with:
name: dist-linux-x64
name: dist-linux-amd64
path: dist-linux-x64

- name: Download MacOs artifacts
- name: Download MacOs x64 artifacts
uses: actions/download-artifact@v1
with:
name: dist-macos-x64
name: dist-macos-amd64
path: dist-macos-x64

- name: Download MacOs arm64 artifacts
uses: actions/download-artifact@v1
with:
name: dist-macos-arm64
path: dist-macos-arm64

- name: Download Win32 artifacts
uses: actions/download-artifact@v1
with:
name: dist-win32-x64
name: dist-win32-amd64
path: dist-win32-x64

- name: Set execute permission
run: |
chmod +x dist-linux-x64/longtail
chmod +x dist-macos-x64/longtail
chmod +x dist-macos-arm64/longtail
- name: Rename artifacts
- name: Copy executables
run: |
cp dist-linux-x64/longtail longtail-linux-x64
cp dist-macos-x64/longtail longtail-macos-x64
cp dist-win32-x64/longtail.exe longtail-win32-x64.exe
cp dist-macos-arm64/longtail longtail-macos-arm64
- name: Zip Linux artifacts
uses: montudor/action-zip@v0.1.0
with:
args: zip -qq -r ./linux-x64.zip ./dist-linux-x64

- name: Zip MacOS artifacts
- name: Zip MacOS x64 artifacts
uses: montudor/action-zip@v0.1.0
with:
args: zip -qq -r ./macos-x64.zip ./dist-macos-x64

- name: Zip MacOS arm64 artifacts
uses: montudor/action-zip@v0.1.0
with:
args: zip -qq -r ./macos-arm64.zip ./dist-macos-arm64

- name: Zip Win32 artifacts
uses: montudor/action-zip@v0.1.0
with:
Expand Down Expand Up @@ -142,6 +166,8 @@ jobs:
win32-x64.zip
linux-x64.zip
macos-x64.zip
macos-arm64.zip
longtail-linux-x64
longtail-macos-x64
longtail-macos-arm64
longtail-win32-x64.exe
11 changes: 11 additions & 0 deletions .github/workflows/master-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@ jobs:
strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
arch: [amd64]
include:
- os: macos-latest
arch: arm64

runs-on: ${{matrix.os}}

Expand All @@ -24,6 +28,7 @@ jobs:
uses: actions/checkout@v3

- name: test
if: ${{ matrix.arch == 'amd64' }}
run: |
pushd ./longtaillib
go test .
Expand All @@ -37,9 +42,15 @@ jobs:
pushd ./remotestore
go test .
popd
env:
GOARCH: ${{matrix.arch}}
CGO_ENABLED: 1

- name: build cmd
run: |
pushd ./cmd/longtail
go build .
popd
env:
GOARCH: ${{matrix.arch}}
CGO_ENABLED: 1
11 changes: 11 additions & 0 deletions .github/workflows/pr-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ jobs:
strategy:
matrix:
os: [ubuntu-20.04, macos-latest, windows-latest]
arch: [amd64]
include:
- os: macos-latest
arch: arm64

runs-on: ${{matrix.os}}

Expand All @@ -33,6 +37,7 @@ jobs:
uses: actions/checkout@v3

- name: test
if: ${{ matrix.arch == 'amd64' }}
run: |
pushd ./longtaillib
go test .
Expand All @@ -46,9 +51,15 @@ jobs:
pushd ./remotestore
go test .
popd
env:
GOARCH: ${{matrix.arch}}
CGO_ENABLED: 1

- name: build cmd
run: |
pushd ./cmd/longtail
go build .
popd
env:
GOARCH: ${{matrix.arch}}
CGO_ENABLED: 1
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,9 @@
##
- **NEW** added builds for Arm64 flavours
- meow hash option is not supported on Arm64 and will return 0 if called
- blake2 hash option is not supported on Arm64 and will return 0 if called
- `macos-arm64.zip` and `longtail-macos-arm64` artifacts are produced when creating a release
- **UPDATED** Updated to longtail 0.4.0
- **UPDATED** Updated all golang dependencies
- **UPDATED** Updated Github Actions Linux runners to 20.04

Expand Down
16 changes: 14 additions & 2 deletions longtaillib/longtail/include/src/longtail.h
Original file line number Diff line number Diff line change
Expand Up @@ -1555,7 +1555,7 @@ struct Longtail_StoreIndex
uint32_t* m_ChunkCount; // Total number of chunks across all blocks - chunk hashes may occur more than once
TLongtail_Hash* m_BlockHashes; // [] m_BlockHashes is the hash of each block
TLongtail_Hash* m_ChunkHashes; // [] For each m_BlockChunkCount[n] there are n consecutive chunk hashes in m_ChunkHashes[]
uint32_t* m_BlockChunksOffsets; // [] m_BlockChunksOffsets[n] is the offset in m_ChunkBlockCount[] and m_ChunkHashes[]
uint32_t* m_BlockChunksOffsets; // [] m_BlockChunksOffsets[n] is the offset in m_ChunkSizes[] and m_ChunkHashes[]
uint32_t* m_BlockChunkCounts; // [] m_BlockChunkCounts[n] is number of chunks in block m_BlockHash[n]
uint32_t* m_BlockTags; // [] m_BlockTags is the tag for each block
uint32_t* m_ChunkSizes; // [] m_ChunkSizes is the size of each chunk
Expand Down Expand Up @@ -1632,7 +1632,19 @@ LONGTAIL_EXPORT int Longtail_ValidateStore(
* @param[in] store_index The source index to copy from
* @return An initialized struct Longtail_StoreIndex copied from @p store_index, or 0 on bad parameters / out of memory
*/
LONGTAIL_EXPORT struct Longtail_StoreIndex* Longtail_CopyStoreIndex(struct Longtail_StoreIndex* store_index);
LONGTAIL_EXPORT struct Longtail_StoreIndex* Longtail_CopyStoreIndex(const struct Longtail_StoreIndex* store_index);

/*! @brief Splits a Longtail_StoreIndex into multiple store indexes
*
* Splits a Longtail_StoreIndex into multiple store indexes with each having a max size of split_size.
* *
* @param[in] store_index The source index to split from
* @param[in] split_size The maximum size each resulting store index may have
* @param[out] split_size A pointer to a pointer that will contain a pointer to an array of store index pointers
* @param[out] out_count A pointer that will contain the number of store indexes in the resulting out_store_indexes array
* @return Return code (errno style), zero on success.
*/
LONGTAIL_EXPORT int Longtail_SplitStoreIndex(struct Longtail_StoreIndex* store_index, size_t split_size, struct Longtail_StoreIndex*** out_store_indexes, uint64_t* out_count);

/*! @brief Writes a struct Longtail_StoreIndex to a byte buffer.
*
Expand Down
Binary file added longtaillib/longtail/liblongtail_darwin_arm64.a
Binary file not shown.
Binary file modified longtaillib/longtail/liblongtail_darwin_x64.a
Binary file not shown.
Binary file modified longtaillib/longtail/liblongtail_linux_x64.a
Binary file not shown.
Binary file modified longtaillib/longtail/liblongtail_win32_x64.a
Binary file not shown.
4 changes: 4 additions & 0 deletions longtaillib/longtail_darwin_arm64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package longtaillib

// #cgo LDFLAGS: -L${SRCDIR}/longtail -llongtail_darwin_arm64 -lm
import "C"
75 changes: 75 additions & 0 deletions longtailstorelib/fsstore_unix_arm64.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// +build aix darwin dragonfly freebsd js,wasm linux nacl netbsd openbsd solaris

package longtailstorelib

import (
"fmt"
"syscall"
"time"

"github.com/pkg/errors"
)

type Lock struct {
filename string
fd int
}

func NewFileLock(filename string) *Lock {
return &Lock{filename: filename}
}

func (l *Lock) Lock() error {
const fname = "Lock.open"
if err := l.open(); err != nil {
return errors.Wrap(err, fname)
}
return syscall.Flock(l.fd, syscall.LOCK_EX)
}

func (l *Lock) Unlock() error {
const fname = "Lock.Unlock"
err := syscall.Close(l.fd)
if err != nil {
return errors.Wrap(err, fname)
}
return nil
}

func (l *Lock) LockWithTimeout(timeout time.Duration) error {
const fname = "Lock.LockWithTimeout"
err := l.open()
if err != nil {
return errors.Wrap(err, fname)
}
result := make(chan error)
cancel := make(chan struct{})
go func() {
err := syscall.Flock(l.fd, syscall.LOCK_EX)
select {
case <-cancel:
// Timed out, cleanup if necessary.
syscall.Flock(l.fd, syscall.LOCK_UN)
syscall.Close(l.fd)
case result <- err:
}
}()
select {
case err := <-result:
return errors.Wrap(err, fname)
case <-time.After(timeout):
close(cancel)
err := fmt.Errorf("Retry timed out for lock file `%s`, waited %s", l.filename, timeout.String())
return errors.Wrap(err, fname)
}
}

func (l *Lock) open() error {
const fname = "Lock.open"
fd, err := syscall.Open(l.filename, syscall.O_CREAT|syscall.O_RDONLY, 0600)
if err != nil {
return errors.Wrap(err, fname)
}
l.fd = fd
return nil
}

0 comments on commit 6076be7

Please sign in to comment.