bzzz/test/integration/ucxi_dht_integration_test.go

// Integration Tests for Issue 009: UCXI + DHT Encryption + Search
// These tests validate the complete integration between UCXI HTTP server,
// encrypted DHT storage, and search functionality with proper UCXL addressing.

package integration

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"chorus.services/bzzz/pkg/config"
	"chorus.services/bzzz/pkg/crypto"
	"chorus.services/bzzz/pkg/dht"
	"chorus.services/bzzz/pkg/ucxi"
	"chorus.services/bzzz/pkg/ucxl"
)

// UCXIDHTIntegrationTestSuite provides comprehensive testing for UCXI + DHT + Encryption
type UCXIDHTIntegrationTestSuite struct {
	ctx           context.Context
	config        *config.Config
	keyManager    *crypto.KeyManager
	dhtStorage    dht.DHT
	ucxiServer    *ucxi.Server
	httpServer    *httptest.Server
	testData      map[string][]byte
	testAddresses []string
}

func TestUCXIDHTIntegration(t *testing.T) {
	suite := NewUCXIDHTIntegrationTestSuite(t)
	defer suite.Cleanup()

	t.Run("PutGetDelete_ValidAddresses", suite.TestPutGetDeleteValidAddresses)
	t.Run("Encryption_Decryption_RoleBased", suite.TestEncryptionDecryptionRoleBased)
	t.Run("Search_AgentRoleProjectTaskFilters", suite.TestSearchWithFilters)
	t.Run("TemporalAddressing_Navigation", suite.TestTemporalAddressing)
	t.Run("InvalidAddress_Returns_UCXL400", suite.TestInvalidAddressValidation)
	t.Run("ConcurrentOperations_ThreadSafety", suite.TestConcurrentOperations)
	t.Run("LargePayload_StorageRetrieval", suite.TestLargePayloadHandling)
	t.Run("TTL_Expiration_Cleanup", suite.TestTTLExpirationCleanup)
}

func NewUCXIDHTIntegrationTestSuite(t *testing.T) *UCXIDHTIntegrationTestSuite {
	ctx := context.Background()

	// Initialize test configuration
	cfg := &config.Config{
		Security: config.SecurityConfig{
			AuditLogging:     true,
			KeyRotationDays:  30,
			MaxKeyAge:        time.Hour * 24 * 365,
			RequireKeyEscrow: true,
		},
		Roles: []config.Role{
			{Name: "developer", Permissions: []string{"read", "write"}},
			{Name: "admin", Permissions: []string{"read", "write", "delete", "admin"}},
			{Name: "viewer", Permissions: []string{"read"}},
		},
	}

	// Initialize key manager
	keyManager, err := crypto.NewKeyManager(cfg, crypto.NewInMemoryKeyStore())
	require.NoError(t, err, "Failed to create key manager")

	// Initialize mock DHT storage
	dhtStorage := dht.NewMockDHT()

	// Initialize encrypted storage layer
	encryptedStorage, err := dht.NewEncryptedStorage(dhtStorage, keyManager)
	require.NoError(t, err, "Failed to create encrypted storage")

	// Initialize UCXI server
	ucxiServer, err := ucxi.NewServer(encryptedStorage)
	require.NoError(t, err, "Failed to create UCXI server")

	// Create HTTP test server
	httpServer := httptest.NewServer(ucxiServer)

	// Prepare test data
	testData := map[string][]byte{
		"simple_config":    []byte(`{"version": "1.0", "enabled": true}`),
		"user_data":       []byte(`{"name": "John Doe", "role": "developer", "team": "backend"}`),
		"large_document":  bytes.Repeat([]byte("test data "), 1000),
		"json_array":      []byte(`[{"id": 1, "value": "first"}, {"id": 2, "value": "second"}]`),
		"binary_data":     []byte{0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x57, 0x6f, 0x72, 0x6c, 0x64},
	}

	// Generate test addresses with different patterns
	testAddresses := []string{
		"ucxl://agent1:developer@project1:task1/*^",
		"ucxl://admin:admin@bzzz:config/cluster/nodes*^",
		"ucxl://user1:viewer@docs:read/api/v1*^",
		"ucxl://service:developer@microservice:deploy/staging*^",
		"ucxl://monitor:admin@system:health/metrics*^",
	}

	return &UCXIDHTIntegrationTestSuite{
		ctx:           ctx,
		config:        cfg,
		keyManager:    keyManager,
		dhtStorage:    dhtStorage,
		ucxiServer:    ucxiServer,
		httpServer:    httpServer,
		testData:      testData,
		testAddresses: testAddresses,
	}
}

func (suite *UCXIDHTIntegrationTestSuite) Cleanup() {
	suite.httpServer.Close()
}

// TestPutGetDeleteValidAddresses tests the complete PUT/GET/DELETE cycle with valid UCXL addresses
func (suite *UCXIDHTIntegrationTestSuite) TestPutGetDeleteValidAddresses(t *testing.T) {
	for i, address := range suite.testAddresses {
		testDataKey := []string{"simple_config", "user_data", "json_array", "binary_data", "large_document"}[i%5]
		testData := suite.testData[testDataKey]

		t.Run(fmt.Sprintf("Address_%d_%s", i, strings.ReplaceAll(address, ":", "_")), func(t *testing.T) {
			// 1. PUT: Store data at the address
			putResp, err := http.Post(
				fmt.Sprintf("%s/put/%s", suite.httpServer.URL, address),
				"application/octet-stream",
				bytes.NewReader(testData),
			)
			require.NoError(t, err, "PUT request failed")
			require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed")
			putResp.Body.Close()

			// 2. GET: Retrieve data from the address
			getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, address))
			require.NoError(t, err, "GET request failed")
			require.Equal(t, http.StatusOK, getResp.StatusCode, "GET should succeed")

			var getBody bytes.Buffer
			_, err = getBody.ReadFrom(getResp.Body)
			require.NoError(t, err, "Failed to read GET response body")
			getResp.Body.Close()

			assert.Equal(t, testData, getBody.Bytes(), "Retrieved data should match stored data")

			// 3. DELETE: Remove data from the address
			delReq, err := http.NewRequest("DELETE", fmt.Sprintf("%s/delete/%s", suite.httpServer.URL, address), nil)
			require.NoError(t, err, "Failed to create DELETE request")

			client := &http.Client{}
			delResp, err := client.Do(delReq)
			require.NoError(t, err, "DELETE request failed")
			require.Equal(t, http.StatusOK, delResp.StatusCode, "DELETE should succeed")
			delResp.Body.Close()

			// 4. GET after DELETE should return 404
			getAfterDelResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, address))
			require.NoError(t, err, "GET after DELETE request failed")
			assert.Equal(t, http.StatusNotFound, getAfterDelResp.StatusCode, "GET after DELETE should return 404")
			getAfterDelResp.Body.Close()
		})
	}
}

// TestEncryptionDecryptionRoleBased tests role-based encryption and decryption
func (suite *UCXIDHTIntegrationTestSuite) TestEncryptionDecryptionRoleBased(t *testing.T) {
	testCases := []struct {
		name        string
		address     string
		role        string
		data        []byte
		expectError bool
	}{
		{
			name:        "Developer_ReadWrite",
			address:     "ucxl://dev1:developer@project1:task1/*^",
			role:        "developer",
			data:        []byte(`{"secret": "developer_data", "level": "standard"}`),
			expectError: false,
		},
		{
			name:        "Admin_FullAccess",
			address:     "ucxl://admin1:admin@system:config/*^",
			role:        "admin",
			data:        []byte(`{"secret": "admin_data", "level": "restricted"}`),
			expectError: false,
		},
		{
			name:        "Viewer_ReadOnly",
			address:     "ucxl://viewer1:viewer@docs:read/*^",
			role:        "viewer",
			data:        []byte(`{"public": "viewer_data", "level": "public"}`),
			expectError: false,
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			// Parse address to extract role information
			parsedAddr, err := ucxl.ParseUCXLAddress(tc.address)
			require.NoError(t, err, "Failed to parse test address")
			assert.Equal(t, tc.role, parsedAddr.Role, "Role should match expected value")

			// Store encrypted data
			putResp, err := http.Post(
				fmt.Sprintf("%s/put/%s", suite.httpServer.URL, tc.address),
				"application/json",
				bytes.NewReader(tc.data),
			)
			require.NoError(t, err, "PUT request failed")

			if tc.expectError {
				assert.NotEqual(t, http.StatusOK, putResp.StatusCode, "PUT should fail for invalid role")
				putResp.Body.Close()
				return
			}

			require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed for valid role")
			putResp.Body.Close()

			// Retrieve and verify decrypted data
			getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, tc.address))
			require.NoError(t, err, "GET request failed")
			require.Equal(t, http.StatusOK, getResp.StatusCode, "GET should succeed")

			var retrieved []byte
			var getBody bytes.Buffer
			_, err = getBody.ReadFrom(getResp.Body)
			require.NoError(t, err, "Failed to read response")
			retrieved = getBody.Bytes()
			getResp.Body.Close()

			assert.Equal(t, tc.data, retrieved, "Decrypted data should match original")

			// Verify data is actually encrypted in storage
			directValue, err := suite.dhtStorage.GetValue(suite.ctx, tc.address)
			if err == nil {
				// Direct storage value should be different from original (encrypted)
				assert.NotEqual(t, tc.data, directValue, "Data should be encrypted in storage")
			}
		})
	}
}

// TestSearchWithFilters tests search functionality with agent/role/project/task filters
func (suite *UCXIDHTIntegrationTestSuite) TestSearchWithFilters(t *testing.T) {
	// First, populate storage with multiple entries for searching
	testEntries := []struct {
		address string
		data    []byte
	}{
		{"ucxl://alice:developer@projectA:feature1/*^", []byte(`{"author": "alice", "type": "feature"}`)},
		{"ucxl://bob:developer@projectA:bugfix2/*^", []byte(`{"author": "bob", "type": "bugfix"}`)},
		{"ucxl://charlie:admin@projectB:config3/*^", []byte(`{"author": "charlie", "type": "config"}`)},
		{"ucxl://alice:developer@projectB:feature4/*^", []byte(`{"author": "alice", "type": "feature"}`)},
		{"ucxl://diana:viewer@projectA:read5/*^", []byte(`{"author": "diana", "type": "read"}`)},
	}

	// Store all test entries
	for _, entry := range testEntries {
		putResp, err := http.Post(
			fmt.Sprintf("%s/put/%s", suite.httpServer.URL, entry.address),
			"application/json",
			bytes.NewReader(entry.data),
		)
		require.NoError(t, err, "Failed to store test entry")
		require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed")
		putResp.Body.Close()
	}

	searchTestCases := []struct {
		name           string
		searchPattern  string
		expectedCount  int
		expectedAgents []string
	}{
		{
			name:           "Search_All_ProjectA",
			searchPattern:  "ucxl://*:*@projectA:*/*",
			expectedCount:  3,
			expectedAgents: []string{"alice", "bob", "diana"},
		},
		{
			name:           "Search_Developer_Role",
			searchPattern:  "ucxl://*:developer@*:*/*",
			expectedCount:  3,
			expectedAgents: []string{"alice", "bob", "alice"}, // alice appears twice
		},
		{
			name:           "Search_Alice_Agent",
			searchPattern:  "ucxl://alice:*@*:*/*",
			expectedCount:  2,
			expectedAgents: []string{"alice", "alice"},
		},
		{
			name:           "Search_Admin_ProjectB",
			searchPattern:  "ucxl://*:admin@projectB:*/*",
			expectedCount:  1,
			expectedAgents: []string{"charlie"},
		},
		{
			name:           "Search_Feature_Tasks",
			searchPattern:  "ucxl://*:*@*:feature*/*",
			expectedCount:  2,
			expectedAgents: []string{"alice", "alice"},
		},
	}

	for _, tc := range searchTestCases {
		t.Run(tc.name, func(t *testing.T) {
			// Perform search using UCXI discover endpoint
			searchResp, err := http.Get(fmt.Sprintf("%s/discover?pattern=%s", suite.httpServer.URL, tc.searchPattern))
			require.NoError(t, err, "Search request failed")
			require.Equal(t, http.StatusOK, searchResp.StatusCode, "Search should succeed")

			var searchResults map[string]interface{}
			err = json.NewDecoder(searchResp.Body).Decode(&searchResults)
			require.NoError(t, err, "Failed to decode search results")
			searchResp.Body.Close()

			// Verify search results
			results, ok := searchResults["results"].([]interface{})
			require.True(t, ok, "Search results should contain results array")
			assert.Len(t, results, tc.expectedCount, "Should find expected number of results")

			// Verify that expected agents are found
			foundAgents := make(map[string]int)
			for _, result := range results {
				resultMap := result.(map[string]interface{})
				address := resultMap["address"].(string)
				parsed, err := ucxl.ParseUCXLAddress(address)
				require.NoError(t, err, "Should be able to parse result address")
				foundAgents[parsed.Agent]++
			}

			for _, expectedAgent := range tc.expectedAgents {
				assert.Greater(t, foundAgents[expectedAgent], 0, "Should find expected agent: %s", expectedAgent)
			}
		})
	}
}

// TestTemporalAddressing tests temporal navigation functionality
func (suite *UCXIDHTIntegrationTestSuite) TestTemporalAddressing(t *testing.T) {
	baseAddress := "ucxl://agent1:developer@project1:task1/*"
	
	// Create multiple versions
	versions := []struct {
		address string
		data    []byte
		version string
	}{
		{baseAddress + "v1", []byte(`{"version": 1, "data": "first version"}`), "v1"},
		{baseAddress + "v2", []byte(`{"version": 2, "data": "second version"}`), "v2"},
		{baseAddress + "v3", []byte(`{"version": 3, "data": "third version"}`), "v3"},
		{baseAddress + "^", []byte(`{"version": 999, "data": "latest version"}`), "latest"},
	}

	// Store all versions
	for _, v := range versions {
		putResp, err := http.Post(
			fmt.Sprintf("%s/put/%s", suite.httpServer.URL, v.address),
			"application/json",
			bytes.NewReader(v.data),
		)
		require.NoError(t, err, "Failed to store version")
		require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed")
		putResp.Body.Close()
	}

	// Test temporal navigation
	navigationTests := []struct {
		name       string
		address    string
		expectData string
	}{
		{
			name:       "Latest_Version",
			address:    baseAddress + "^",
			expectData: "latest version",
		},
		{
			name:       "Specific_Version_v2",
			address:    baseAddress + "v2",
			expectData: "second version",
		},
		{
			name:       "Backward_Navigation",
			address:    baseAddress + "^-1", // Latest minus 1
			expectData: "third version",
		},
	}

	for _, nt := range navigationTests {
		t.Run(nt.name, func(t *testing.T) {
			getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, nt.address))
			require.NoError(t, err, "GET request failed")
			require.Equal(t, http.StatusOK, getResp.StatusCode, "GET should succeed")

			var result map[string]interface{}
			err = json.NewDecoder(getResp.Body).Decode(&result)
			require.NoError(t, err, "Failed to decode response")
			getResp.Body.Close()

			assert.Contains(t, result["data"], nt.expectData, "Should retrieve correct version")
		})
	}
}

// TestInvalidAddressValidation tests that invalid addresses return proper UCXL-400 codes
func (suite *UCXIDHTIntegrationTestSuite) TestInvalidAddressValidation(t *testing.T) {
	invalidAddresses := []struct {
		address string
		reason  string
	}{
		{"invalid-address", "missing scheme"},
		{"ucxl://", "empty address components"},
		{"ucxl://:role@project:task/*", "empty agent"},
		{"ucxl://agent:@project:task/*", "empty role"},
		{"ucxl://agent:role@:task/*", "empty project"},
		{"ucxl://agent:role@project:/*", "empty task"},
		{"http://agent:role@project:task/*", "wrong scheme"},
		{"ucxl://ag@ent:role@project:task/*", "invalid characters"},
		{"ucxl://agent:role@project:task", "missing temporal segment"},
	}

	testData := []byte(`{"test": "data"}`)

	for _, ia := range invalidAddresses {
		t.Run(fmt.Sprintf("Invalid_%s", strings.ReplaceAll(ia.reason, " ", "_")), func(t *testing.T) {
			// Test PUT with invalid address
			putResp, err := http.Post(
				fmt.Sprintf("%s/put/%s", suite.httpServer.URL, ia.address),
				"application/json",
				bytes.NewReader(testData),
			)
			require.NoError(t, err, "PUT request should complete")
			assert.Equal(t, http.StatusBadRequest, putResp.StatusCode, 
				"PUT with invalid address should return 400: %s", ia.reason)
			putResp.Body.Close()

			// Test GET with invalid address
			getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, ia.address))
			require.NoError(t, err, "GET request should complete")
			assert.Equal(t, http.StatusBadRequest, getResp.StatusCode, 
				"GET with invalid address should return 400: %s", ia.reason)
			getResp.Body.Close()
		})
	}
}

// TestConcurrentOperations tests thread safety under concurrent access
func (suite *UCXIDHTIntegrationTestSuite) TestConcurrentOperations(t *testing.T) {
	const numGoroutines = 10
	const operationsPerGoroutine = 50

	errChan := make(chan error, numGoroutines*operationsPerGoroutine)
	doneChan := make(chan bool, numGoroutines)

	// Start concurrent operations
	for i := 0; i < numGoroutines; i++ {
		go func(goroutineID int) {
			defer func() { doneChan <- true }()

			for j := 0; j < operationsPerGoroutine; j++ {
				address := fmt.Sprintf("ucxl://worker%d:developer@project:task%d/*^", goroutineID, j)
				testData := []byte(fmt.Sprintf(`{"worker": %d, "operation": %d}`, goroutineID, j))

				// PUT operation
				putResp, err := http.Post(
					fmt.Sprintf("%s/put/%s", suite.httpServer.URL, address),
					"application/json",
					bytes.NewReader(testData),
				)
				if err != nil {
					errChan <- fmt.Errorf("PUT failed for worker %d operation %d: %v", goroutineID, j, err)
					continue
				}
				putResp.Body.Close()

				if putResp.StatusCode != http.StatusOK {
					errChan <- fmt.Errorf("PUT returned %d for worker %d operation %d", 
						putResp.StatusCode, goroutineID, j)
					continue
				}

				// GET operation
				getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, address))
				if err != nil {
					errChan <- fmt.Errorf("GET failed for worker %d operation %d: %v", goroutineID, j, err)
					continue
				}
				getResp.Body.Close()

				if getResp.StatusCode != http.StatusOK {
					errChan <- fmt.Errorf("GET returned %d for worker %d operation %d", 
						getResp.StatusCode, goroutineID, j)
					continue
				}
			}
		}(i)
	}

	// Wait for all goroutines to complete
	for i := 0; i < numGoroutines; i++ {
		<-doneChan
	}
	close(errChan)

	// Check for errors
	var errors []error
	for err := range errChan {
		errors = append(errors, err)
	}

	if len(errors) > 0 {
		t.Errorf("Concurrent operations failed with %d errors:", len(errors))
		for _, err := range errors[:min(10, len(errors))] { // Show first 10 errors
			t.Errorf("  - %v", err)
		}
	}

	// Verify final storage state
	stats := suite.dhtStorage.GetStats()
	expectedKeys := numGoroutines * operationsPerGoroutine
	assert.Equal(t, expectedKeys, stats.TotalKeys, "Should have stored all keys successfully")
}

// TestLargePayloadHandling tests storage and retrieval of large payloads
func (suite *UCXIDHTIntegrationTestSuite) TestLargePayloadHandling(t *testing.T) {
	payloadSizes := []struct {
		name string
		size int
	}{
		{"1KB", 1024},
		{"10KB", 10 * 1024},
		{"100KB", 100 * 1024},
		{"1MB", 1024 * 1024},
	}

	for _, ps := range payloadSizes {
		t.Run(fmt.Sprintf("Payload_%s", ps.name), func(t *testing.T) {
			// Generate large payload
			payload := make([]byte, ps.size)
			for i := range payload {
				payload[i] = byte(i % 256)
			}

			address := fmt.Sprintf("ucxl://tester:developer@large:payload_%s/*^", ps.name)

			start := time.Now()

			// Store large payload
			putResp, err := http.Post(
				fmt.Sprintf("%s/put/%s", suite.httpServer.URL, address),
				"application/octet-stream",
				bytes.NewReader(payload),
			)
			require.NoError(t, err, "PUT request failed")
			require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed")
			putResp.Body.Close()

			putTime := time.Since(start)

			// Retrieve large payload
			start = time.Now()
			getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, address))
			require.NoError(t, err, "GET request failed")
			require.Equal(t, http.StatusOK, getResp.StatusCode, "GET should succeed")

			var retrieved bytes.Buffer
			_, err = retrieved.ReadFrom(getResp.Body)
			require.NoError(t, err, "Failed to read response")
			getResp.Body.Close()

			getTime := time.Since(start)

			// Verify payload integrity
			assert.Equal(t, payload, retrieved.Bytes(), "Retrieved payload should match original")

			t.Logf("Payload %s: PUT=%v, GET=%v, Size=%d bytes", 
				ps.name, putTime, getTime, len(payload))

			// Performance assertions (reasonable thresholds for test environment)
			assert.Less(t, putTime, time.Second*10, "PUT should complete within 10 seconds")
			assert.Less(t, getTime, time.Second*10, "GET should complete within 10 seconds")
		})
	}
}

// TestTTLExpirationCleanup tests TTL-based expiration and cleanup
func (suite *UCXIDHTIntegrationTestSuite) TestTTLExpirationCleanup(t *testing.T) {
	// This test requires a mock DHT that supports TTL
	// For now, we'll test the API behavior and assume the underlying storage respects TTL
	
	shortTTLAddress := "ucxl://temp:developer@project:shortlived/*^"
	testData := []byte(`{"ttl": "short", "data": "should expire soon"}`)

	// Store data with short TTL (this would need to be configured in the storage layer)
	putResp, err := http.Post(
		fmt.Sprintf("%s/put/%s", suite.httpServer.URL, shortTTLAddress),
		"application/json",
		bytes.NewReader(testData),
	)
	require.NoError(t, err, "PUT request failed")
	require.Equal(t, http.StatusOK, putResp.StatusCode, "PUT should succeed")
	putResp.Body.Close()

	// Immediate retrieval should work
	getResp, err := http.Get(fmt.Sprintf("%s/get/%s", suite.httpServer.URL, shortTTLAddress))
	require.NoError(t, err, "GET request failed")
	require.Equal(t, http.StatusOK, getResp.StatusCode, "GET should succeed immediately")
	getResp.Body.Close()

	// Test health endpoint to ensure server is responsive
	healthResp, err := http.Get(fmt.Sprintf("%s/health", suite.httpServer.URL))
	require.NoError(t, err, "Health check failed")
	require.Equal(t, http.StatusOK, healthResp.StatusCode, "Health check should pass")
	
	var healthData map[string]interface{}
	err = json.NewDecoder(healthResp.Body).Decode(&healthData)
	require.NoError(t, err, "Failed to decode health response")
	healthResp.Body.Close()

	assert.Equal(t, "healthy", healthData["status"], "Server should be healthy")
	
	t.Logf("TTL expiration test completed - would need real TTL implementation for full testing")
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}