More work with peer erroring

This commit is contained in:
Chad Retz 2019-02-22 12:40:02 -06:00
parent 22189e051d
commit 8a023d2017
8 changed files with 282 additions and 112 deletions

View File

@ -4,35 +4,61 @@ import (
"context" "context"
"fmt" "fmt"
"sync" "sync"
"time"
) )
type Gun struct { type Gun struct {
peers []Peer peers []*gunPeer
storage Storage storage Storage
soulGen func() string soulGen func() string
peerErrorHandler func(*ErrPeer) peerErrorHandler func(*ErrPeer)
peerSleepOnError time.Duration
messageIDPutListeners map[string]chan<- *MessageReceived messageIDPutListeners map[string]chan<- *MessageReceived
messageIDPutListenersLock sync.RWMutex messageIDPutListenersLock sync.RWMutex
} }
type Config struct { type Config struct {
Peers []Peer PeerURLs []string
Storage Storage Storage Storage
SoulGen func() string SoulGen func() string
PeerErrorHandler func(*ErrPeer) PeerErrorHandler func(*ErrPeer)
PeerSleepOnError time.Duration
} }
func New(config Config) *Gun { const DefaultPeerSleepOnError = 30 * time.Second
func New(ctx context.Context, config Config) (*Gun, error) {
g := &Gun{ g := &Gun{
peers: make([]Peer, len(config.Peers)), peers: make([]*gunPeer, len(config.PeerURLs)),
storage: config.Storage, storage: config.Storage,
soulGen: config.SoulGen, soulGen: config.SoulGen,
peerErrorHandler: config.PeerErrorHandler, peerErrorHandler: config.PeerErrorHandler,
peerSleepOnError: config.PeerSleepOnError,
messageIDPutListeners: map[string]chan<- *MessageReceived{}, messageIDPutListeners: map[string]chan<- *MessageReceived{},
} }
// Copy over peers // Create all the peers
copy(g.peers, config.Peers) sleepOnError := config.PeerSleepOnError
if sleepOnError == 0 {
sleepOnError = DefaultPeerSleepOnError
}
var err error
for i := 0; i < len(config.PeerURLs) && err == nil; i++ {
peerURL := config.PeerURLs[i]
connPeer := func() (Peer, error) { return NewPeer(ctx, peerURL) }
if g.peers[i], err = newGunPeer(connPeer, sleepOnError); err != nil {
err = fmt.Errorf("Failed connecting to peer %v: %v", peerURL, err)
}
}
// If there was an error, we need to close what we did create
if err != nil {
for _, peer := range g.peers {
if peer != nil {
peer.Close()
}
}
return nil, err
}
// Set defaults // Set defaults
if g.storage == nil { if g.storage == nil {
g.storage = &StorageInMem{} g.storage = &StorageInMem{}
@ -42,27 +68,7 @@ func New(config Config) *Gun {
} }
// Start receiving // Start receiving
g.startReceiving() g.startReceiving()
return g return g, nil
}
// To note: Fails on even one peer failure (otherwise, do this yourself). May connect to
// some peers temporarily until first failure, but closes them all on failure
func NewFromPeerURLs(ctx context.Context, peerURLs ...string) (g *Gun, err error) {
c := Config{Peers: make([]Peer, len(peerURLs))}
for i := 0; i < len(peerURLs) && err == nil; i++ {
if c.Peers[i], err = NewPeer(ctx, peerURLs[i]); err != nil {
err = fmt.Errorf("Failed connecting to peer %v: %v", peerURLs[i], err)
}
}
if err != nil {
for _, peer := range c.Peers {
if peer != nil {
peer.Close()
}
}
return
}
return New(c), nil
} }
func (g *Gun) Close() error { func (g *Gun) Close() error {
@ -85,7 +91,7 @@ func (g *Gun) Send(ctx context.Context, msg *Message) <-chan *ErrPeer {
return g.send(ctx, msg, nil) return g.send(ctx, msg, nil)
} }
func (g *Gun) send(ctx context.Context, msg *Message, ignorePeer Peer) <-chan *ErrPeer { func (g *Gun) send(ctx context.Context, msg *Message, ignorePeer *gunPeer) <-chan *ErrPeer {
ch := make(chan *ErrPeer, len(g.peers)) ch := make(chan *ErrPeer, len(g.peers))
// Everything async // Everything async
go func() { go func() {
@ -96,9 +102,10 @@ func (g *Gun) send(ctx context.Context, msg *Message, ignorePeer Peer) <-chan *E
continue continue
} }
wg.Add(1) wg.Add(1)
go func(peer Peer) { go func(peer *gunPeer) {
defer wg.Done() defer wg.Done()
if err := peer.Send(ctx, msg); err != nil { // Just do nothing if the peer is bad and we couldn't send
if _, err := peer.send(ctx, msg); err != nil {
peerErr := &ErrPeer{err, peer} peerErr := &ErrPeer{err, peer}
go g.onPeerError(peerErr) go g.onPeerError(peerErr)
ch <- peerErr ch <- peerErr
@ -112,24 +119,35 @@ func (g *Gun) send(ctx context.Context, msg *Message, ignorePeer Peer) <-chan *E
func (g *Gun) startReceiving() { func (g *Gun) startReceiving() {
for _, peer := range g.peers { for _, peer := range g.peers {
go func(peer Peer) { go func(peer *gunPeer) {
for msgOrErr := range peer.Receive() { // TDO: some kind of overall context is probably needed
if msgOrErr.Err != nil { ctx, cancelFn := context.WithCancel(context.TODO())
go g.onPeerError(&ErrPeer{msgOrErr.Err, peer}) defer cancelFn()
continue for {
} // We might not be able receive because peer is sleeping from
// See if a listener is around to handle it instead of rebroadcasting // an error happened within or a just-before send error.
msg := &MessageReceived{Message: msgOrErr.Message, Peer: peer} if ok, msgs, err := peer.receive(ctx); !ok {
if msg.Ack != "" && len(msg.Put) > 0 { if err != nil {
g.messageIDPutListenersLock.RLock() go g.onPeerError(&ErrPeer{err, peer})
l := g.messageIDPutListeners[msg.Ack] }
g.messageIDPutListenersLock.RUnlock() // Always sleep at least the err duration
if l != nil { time.Sleep(g.peerSleepOnError)
go safeReceivedMessageSend(l, msg) } else {
continue // Go over each message and see if it needs delivering or rebroadcasting
for _, msg := range msgs {
rMsg := &MessageReceived{Message: msg, peer: peer}
if msg.Ack != "" && len(msg.Put) > 0 {
g.messageIDPutListenersLock.RLock()
l := g.messageIDPutListeners[msg.Ack]
g.messageIDPutListenersLock.RUnlock()
if l != nil {
go safeReceivedMessageSend(l, rMsg)
continue
}
}
go g.onUnhandledMessage(rMsg)
} }
} }
go g.onUnhandledMessage(msg)
} }
}(peer) }(peer)
} }
@ -138,7 +156,7 @@ func (g *Gun) startReceiving() {
func (g *Gun) onUnhandledMessage(msg *MessageReceived) { func (g *Gun) onUnhandledMessage(msg *MessageReceived) {
// Unhandled message means rebroadcast // Unhandled message means rebroadcast
// TODO: we need a timeout or global context here... // TODO: we need a timeout or global context here...
g.send(context.TODO(), msg.Message, msg.Peer) g.send(context.TODO(), msg.Message, msg.peer)
} }
func (g *Gun) onPeerError(err *ErrPeer) { func (g *Gun) onPeerError(err *ErrPeer) {

91
gun/gun_peer.go Normal file
View File

@ -0,0 +1,91 @@
package gun
import (
"context"
"sync"
"time"
)
type gunPeer struct {
connPeer func() (Peer, error)
sleepOnErr time.Duration // TODO: would be better as backoff
peer Peer
peerBad bool // If true, don't try anything
peerLock sync.Mutex
}
func newGunPeer(connPeer func() (Peer, error), sleepOnErr time.Duration) (*gunPeer, error) {
p := &gunPeer{connPeer: connPeer, sleepOnErr: sleepOnErr}
var err error
if p.peer, err = connPeer(); err != nil {
return nil, err
}
return p, nil
}
func (g *gunPeer) ID() string {
panic("TODO")
}
func (g *gunPeer) reconnectPeer() (err error) {
g.peerLock.Lock()
defer g.peerLock.Unlock()
if g.peer == nil && g.peerBad {
g.peerBad = false
if g.peer, err = g.connPeer(); err != nil {
g.peerBad = true
time.AfterFunc(g.sleepOnErr, func() { g.reconnectPeer() })
}
}
return
}
// Can be nil peer if currently bad
func (g *gunPeer) connectedPeer() Peer {
g.peerLock.Lock()
defer g.peerLock.Unlock()
return g.peer
}
func (g *gunPeer) markPeerErrored(p Peer) {
g.peerLock.Lock()
defer g.peerLock.Unlock()
if p == g.peer {
g.peer = nil
g.peerBad = true
p.Close()
time.AfterFunc(g.sleepOnErr, func() { g.reconnectPeer() })
}
}
func (g *gunPeer) send(ctx context.Context, msg *Message, moreMsgs ...*Message) (ok bool, err error) {
if p := g.connectedPeer(); p == nil {
return false, nil
} else if err = p.Send(ctx, msg, moreMsgs...); err != nil {
g.markPeerErrored(p)
return false, err
} else {
return true, nil
}
}
func (g *gunPeer) receive(ctx context.Context) (ok bool, msgs []*Message, err error) {
if p := g.connectedPeer(); p == nil {
return false, nil, nil
} else if msgs, err = p.Receive(ctx); err != nil {
g.markPeerErrored(p)
return false, nil, err
} else {
return true, msgs, nil
}
}
func (g *gunPeer) Close() error {
g.peerLock.Lock()
defer g.peerLock.Unlock()
err := g.peer.Close()
g.peer = nil
g.peerBad = false
return err
}

View File

@ -19,5 +19,5 @@ type MessageGetRequest struct {
type MessageReceived struct { type MessageReceived struct {
*Message *Message
Peer Peer peer *gunPeer
} }

View File

@ -2,6 +2,7 @@ package gun
import ( import (
"context" "context"
"encoding/json"
"fmt" "fmt"
"net/url" "net/url"
@ -10,22 +11,18 @@ import (
type ErrPeer struct { type ErrPeer struct {
Err error Err error
Peer Peer peer *gunPeer
} }
func (e *ErrPeer) Error() string { return fmt.Sprintf("Error on peer %v: %v", e.Peer, e.Err) } func (e *ErrPeer) Error() string { return fmt.Sprintf("Error on peer %v: %v", e.peer, e.Err) }
type Peer interface { type Peer interface {
Send(ctx context.Context, msg *Message) error Send(ctx context.Context, msg *Message, moreMsgs ...*Message) error
Receive() <-chan *MessageOrError // Chan is closed on first err, when context is closed, or when peer is closed
Receive(ctx context.Context) ([]*Message, error)
Close() error Close() error
} }
type MessageOrError struct {
Message *Message
Err error
}
var PeerURLSchemes = map[string]func(context.Context, *url.URL) (Peer, error){ var PeerURLSchemes = map[string]func(context.Context, *url.URL) (Peer, error){
"ws": func(ctx context.Context, peerUrl *url.URL) (Peer, error) { return NewPeerWebSocket(ctx, peerUrl) }, "ws": func(ctx context.Context, peerUrl *url.URL) (Peer, error) { return NewPeerWebSocket(ctx, peerUrl) },
} }
@ -41,7 +38,7 @@ func NewPeer(ctx context.Context, peerURL string) (Peer, error) {
} }
type PeerWebSocket struct { type PeerWebSocket struct {
*websocket.Conn Underlying *websocket.Conn
} }
func NewPeerWebSocket(ctx context.Context, peerUrl *url.URL) (*PeerWebSocket, error) { func NewPeerWebSocket(ctx context.Context, peerUrl *url.URL) (*PeerWebSocket, error) {
@ -52,10 +49,74 @@ func NewPeerWebSocket(ctx context.Context, peerUrl *url.URL) (*PeerWebSocket, er
return &PeerWebSocket{conn}, nil return &PeerWebSocket{conn}, nil
} }
func (p *PeerWebSocket) Send(ctx context.Context, msg *Message) error { func (p *PeerWebSocket) Send(ctx context.Context, msg *Message, moreMsgs ...*Message) error {
panic("TODO") // If there are more, send all as an array of JSON strings, otherwise just the msg
var toWrite interface{}
if len(moreMsgs) == 0 {
toWrite = msg
} else {
b, err := json.Marshal(msg)
if err != nil {
return err
}
msgs := []string{string(b)}
for _, nextMsg := range moreMsgs {
if b, err = json.Marshal(nextMsg); err != nil {
return err
}
msgs = append(msgs, string(b))
}
toWrite = msgs
}
// Send async so we can wait on context
errCh := make(chan error, 1)
go func() { errCh <- p.Underlying.WriteJSON(toWrite) }()
select {
case err := <-errCh:
return err
case <-ctx.Done():
return ctx.Err()
}
} }
func (p *PeerWebSocket) Receive() <-chan *MessageOrError { func (p *PeerWebSocket) Receive(ctx context.Context) ([]*Message, error) {
panic("TODO") bytsCh := make(chan []byte, 1)
errCh := make(chan error, 1)
go func() {
if _, b, err := p.Underlying.ReadMessage(); err != nil {
errCh <- err
} else {
bytsCh <- b
}
}()
select {
case err := <-errCh:
return nil, err
case <-ctx.Done():
return nil, ctx.Err()
case byts := <-bytsCh:
// If it's a JSON array, it means it's an array of JSON strings, otherwise it's one message
if byts[0] != '[' {
var msg Message
if err := json.Unmarshal(byts, &msg); err != nil {
return nil, err
}
return []*Message{&msg}, nil
}
var jsonStrs []string
if err := json.Unmarshal(byts, &jsonStrs); err != nil {
return nil, err
}
msgs := make([]*Message, len(jsonStrs))
for i, jsonStr := range jsonStrs {
if err := json.Unmarshal([]byte(jsonStr), &(msgs[i])); err != nil {
return nil, err
}
}
return msgs, nil
}
}
func (p *PeerWebSocket) Close() error {
return p.Underlying.Close()
} }

View File

@ -39,7 +39,7 @@ type ValueFetch struct {
// Nil if the value doesn't exist or there's an error // Nil if the value doesn't exist or there's an error
Value *ValueWithState Value *ValueWithState
// Nil when local and sometimes on error // Nil when local and sometimes on error
Peer Peer peer *gunPeer
} }
var ErrNotObject = errors.New("Scoped value not an object") var ErrNotObject = errors.New("Scoped value not an object")
@ -159,7 +159,7 @@ func (s *Scoped) onRemote(ctx context.Context, ch chan *ValueFetch) {
if !ok { if !ok {
return return
} }
f := &ValueFetch{Field: s.field, Peer: msg.Peer} f := &ValueFetch{Field: s.field, peer: msg.peer}
// We asked for a single field, should only get that field or it doesn't exist // We asked for a single field, should only get that field or it doesn't exist
if n := msg.Put[parentSoul]; n != nil && n.Values[s.field] != nil { if n := msg.Put[parentSoul]; n != nil && n.Values[s.field] != nil {
f.Value = &ValueWithState{n.Values[s.field], n.State[s.field]} f.Value = &ValueWithState{n.Values[s.field], n.State[s.field]}
@ -177,7 +177,7 @@ func (s *Scoped) onRemote(ctx context.Context, ch chan *ValueFetch) {
safeValueFetchSend(ch, &ValueFetch{ safeValueFetchSend(ch, &ValueFetch{
Err: peerErr.Err, Err: peerErr.Err,
Field: s.field, Field: s.field,
Peer: peerErr.Peer, peer: peerErr.peer,
}) })
} }
}() }()

View File

@ -11,25 +11,30 @@ import (
"strconv" "strconv"
"testing" "testing"
"github.com/cretz/esgopeta/gun"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
type testContext struct { type testContext struct {
context.Context context.Context
*testing.T *testing.T
Require *require.Assertions Require *require.Assertions
GunJSPort int
} }
func newContext(t *testing.T) (*testContext, context.CancelFunc) { func newContext(t *testing.T) (*testContext, context.CancelFunc) {
return withTestContext(context.Background(), t) return withTestContext(context.Background(), t)
} }
const defaultGunJSPort = 8080
func withTestContext(ctx context.Context, t *testing.T) (*testContext, context.CancelFunc) { func withTestContext(ctx context.Context, t *testing.T) (*testContext, context.CancelFunc) {
ctx, cancelFn := context.WithCancel(ctx) ctx, cancelFn := context.WithCancel(ctx)
return &testContext{ return &testContext{
Context: ctx, Context: ctx,
T: t, T: t,
Require: require.New(t), Require: require.New(t),
GunJSPort: defaultGunJSPort,
}, cancelFn }, cancelFn
} }
@ -50,6 +55,16 @@ func (t *testContext) runJS(script string) []byte {
return out return out
} }
func (t *testContext) runJSWithGun(script string) []byte {
return t.runJS(`
var Gun = require('gun')
const gun = Gun({
peers: ['http://127.0.0.1:` + strconv.Itoa(t.GunJSPort) + `/gun'],
radisk: false
})
` + script)
}
func (t *testContext) startJS(script string) (*bytes.Buffer, *exec.Cmd, context.CancelFunc) { func (t *testContext) startJS(script string) (*bytes.Buffer, *exec.Cmd, context.CancelFunc) {
cmdCtx, cancelFn := context.WithCancel(t) cmdCtx, cancelFn := context.WithCancel(t)
cmd := exec.CommandContext(cmdCtx, "node") cmd := exec.CommandContext(cmdCtx, "node")
@ -62,7 +77,13 @@ func (t *testContext) startJS(script string) (*bytes.Buffer, *exec.Cmd, context.
return &buf, cmd, cancelFn return &buf, cmd, cancelFn
} }
func (t *testContext) startGunServer(port int) { func (t *testContext) startGunJSServer() {
// If we're logging, use a proxy
port := t.GunJSPort
if testing.Verbose() {
t.startGunWebSocketProxyLogger(port, port+1)
port++
}
// Remove entire data folder first // Remove entire data folder first
t.Require.NoError(os.RemoveAll("radata-server")) t.Require.NoError(os.RemoveAll("radata-server"))
t.startJS(` t.startJS(`
@ -71,3 +92,9 @@ func (t *testContext) startGunServer(port int) {
const gun = Gun({web: server, file: 'radata-server'}) const gun = Gun({web: server, file: 'radata-server'})
`) `)
} }
func (t *testContext) newGunConnectedToGunJS() *gun.Gun {
g, err := gun.NewFromPeerURLs(t, "http://127.0.0.1:"+strconv.Itoa(t.GunJSPort)+"/gun")
t.Require.NoError(err)
return g
}

View File

@ -1,20 +1,18 @@
package tests package tests
/* import (
func TestGunGo(t *testing.T) { "testing"
)
func TestGunGetSimple(t *testing.T) {
// Run the server, put in one call, get in another, then check // Run the server, put in one call, get in another, then check
ctx, cancelFn := newContext(t) ctx, cancelFn := newContext(t)
defer cancelFn() defer cancelFn()
ctx.startGunServer(8080) ctx.startGunJSServer()
ctx.startGunWebSocketProxyLogger(8081, 8080)
randStr := randString(30) randStr := randString(30)
ctx.runJS(` // Write w/ JS
var Gun = require('gun') ctx.runJSWithGun(`
const gun = Gun({ gun.get('esgopeta-test').get('TestGunGetSimple').get('some-key').put('` + randStr + `', ack => {
peers: ['http://127.0.0.1:8081/gun'],
radisk: false
})
gun.get('esgopeta-test').get('TestGunJS').get('some-key').put('` + randStr + `', ack => {
if (ack.err) { if (ack.err) {
console.error(ack.err) console.error(ack.err)
process.exit(1) process.exit(1)
@ -22,23 +20,9 @@ func TestGunGo(t *testing.T) {
process.exit(0) process.exit(0)
}) })
`) `)
// out := ctx.runJS(` // Get
// var Gun = require('gun') g := ctx.newGunConnectedToGunJS()
// const gun = Gun({ f := g.Scoped(ctx, "esgopeta-test", "TestGunGet", "some-key").Val(ctx)
// peers: ['http://127.0.0.1:8081/gun'],
// radisk: false
// })
// gun.get('esgopeta-test').get('TestGunJS').get('some-key').once(data => {
// console.log(data)
// process.exit(0)
// })
// `)
// ctx.Require.Equal(randStr, strings.TrimSpace(string(out)))
g, err := gun.NewFromPeerURLs(ctx, "http://127.0.0.1:8081/gun")
ctx.Require.NoError(err)
f := g.Scoped("esgopeta-test", "TestGunJS", "some-key").Val(ctx)
ctx.Require.NoError(f.Err) ctx.Require.NoError(f.Err)
} }
*/

View File

@ -15,15 +15,9 @@ func TestGunJS(t *testing.T) {
// Run the server, put in one call, get in another, then check // Run the server, put in one call, get in another, then check
ctx, cancelFn := newContext(t) ctx, cancelFn := newContext(t)
defer cancelFn() defer cancelFn()
ctx.startGunServer(8080) ctx.startGunJSServer()
ctx.startGunWebSocketProxyLogger(8081, 8080)
randStr := randString(30) randStr := randString(30)
ctx.runJS(` ctx.runJSWithGun(`
var Gun = require('gun')
const gun = Gun({
peers: ['http://127.0.0.1:8081/gun'],
radisk: false
})
gun.get('esgopeta-test').get('TestGunJS').get('some-key').put('` + randStr + `', ack => { gun.get('esgopeta-test').get('TestGunJS').get('some-key').put('` + randStr + `', ack => {
if (ack.err) { if (ack.err) {
console.error(ack.err) console.error(ack.err)
@ -32,12 +26,7 @@ func TestGunJS(t *testing.T) {
process.exit(0) process.exit(0)
}) })
`) `)
out := ctx.runJS(` out := ctx.runJSWithGun(`
var Gun = require('gun')
const gun = Gun({
peers: ['http://127.0.0.1:8081/gun'],
radisk: false
})
gun.get('esgopeta-test').get('TestGunJS').get('some-key').once(data => { gun.get('esgopeta-test').get('TestGunJS').get('some-key').once(data => {
console.log(data) console.log(data)
process.exit(0) process.exit(0)