diff --git a/hackage-server.cabal b/hackage-server.cabal index bd88d1455..89d4badd4 100644 --- a/hackage-server.cabal +++ b/hackage-server.cabal @@ -307,7 +307,9 @@ library Distribution.Server.Features.Browse.Options Distribution.Server.Features.Browse.Parsers Distribution.Server.Features.Core + Distribution.Server.Features.Core.Acid Distribution.Server.Features.Core.State + Distribution.Server.Features.Core.Store Distribution.Server.Features.Core.Backup Distribution.Server.Features.Security Distribution.Server.Features.Security.Backup @@ -320,6 +322,10 @@ library Distribution.Server.Features.Security.SHA256 Distribution.Server.Features.Security.State Distribution.Server.Features.Mirror + Distribution.Server.Features.Mirror.Acid + Distribution.Server.Features.Mirror.Store + Distribution.Server.Features.TarIndexCache.Acid + Distribution.Server.Features.TarIndexCache.Store Distribution.Server.Features.Upload Distribution.Server.Features.Upload.State Distribution.Server.Features.Upload.Backup @@ -370,7 +376,9 @@ library Distribution.Server.Features.Html.HtmlUtilities Distribution.Server.Features.HoogleData Distribution.Server.Features.HaskellPlatform + Distribution.Server.Features.HaskellPlatform.Acid Distribution.Server.Features.HaskellPlatform.State + Distribution.Server.Features.HaskellPlatform.Store Distribution.Server.Features.PackageInfoJSON Distribution.Server.Features.Search Distribution.Server.Features.Search.BM25F @@ -385,8 +393,10 @@ library Distribution.Server.Features.Search.TermBag Distribution.Server.Features.Sitemap.Functions Distribution.Server.Features.Votes + Distribution.Server.Features.Votes.Acid Distribution.Server.Features.Votes.Render Distribution.Server.Features.Votes.State + Distribution.Server.Features.Votes.Store Distribution.Server.Features.Votes.Types Distribution.Server.Features.Vouch Distribution.Server.Features.Vouch.State @@ -403,7 +413,9 @@ library Distribution.Server.Features.Tags.State Distribution.Server.Features.Tags.Types Distribution.Server.Features.AnalyticsPixels + Distribution.Server.Features.AnalyticsPixels.Acid Distribution.Server.Features.AnalyticsPixels.State + Distribution.Server.Features.AnalyticsPixels.Store Distribution.Server.Features.AnalyticsPixels.Types Distribution.Server.Features.UserDetails Distribution.Server.Features.UserDetails.Acid diff --git a/src/Distribution/Server/Features/AnalyticsPixels.hs b/src/Distribution/Server/Features/AnalyticsPixels.hs index fb211cda3..ca9c6d8d0 100644 --- a/src/Distribution/Server/Features/AnalyticsPixels.hs +++ b/src/Distribution/Server/Features/AnalyticsPixels.hs @@ -10,11 +10,11 @@ module Distribution.Server.Features.AnalyticsPixels import Data.Set (Set) +import Distribution.Server.Features.AnalyticsPixels.Acid (acidStore) +import qualified Distribution.Server.Features.AnalyticsPixels.Store as Store import Distribution.Server.Features.AnalyticsPixels.Types -import qualified Distribution.Server.Features.AnalyticsPixels.State as Acid import Distribution.Server.Framework -import Distribution.Server.Framework.BackupRestore import Distribution.Server.Features.Core import Distribution.Server.Features.Upload @@ -53,7 +53,7 @@ initAnalyticsPixelsFeature :: ServerEnv -> UploadFeature -> IO AnalyticsPixelsFeature) initAnalyticsPixelsFeature env@ServerEnv{serverStateDir} = do - dbAnalyticsPixelsState <- analyticsPixelsStateComponent serverStateDir + dbAnalyticsPixelsState <- acidStore serverStateDir analyticsPixelAdded <- newHook analyticsPixelRemoved <- newHook @@ -64,27 +64,9 @@ initAnalyticsPixelsFeature env@ServerEnv{serverStateDir} = do return feature --- | Define the backing store (i.e. database component) -analyticsPixelsStateComponent :: FilePath -> IO (StateComponent AcidState Acid.AnalyticsPixelsState) -analyticsPixelsStateComponent stateDir = do - st <- openLocalStateFrom (stateDir "db" "AnalyticsPixels") Acid.initialAnalyticsPixelsState - return StateComponent { - stateDesc = "Backing store for AnalyticsPixels feature" - , stateHandle = st - , getState = query st Acid.GetAnalyticsPixelsState - , putState = update st . Acid.ReplaceAnalyticsPixelsState - , resetState = analyticsPixelsStateComponent - , backupState = \_ _ -> [] - , restoreState = RestoreBackup { - restoreEntry = error "Unexpected backup entry" - , restoreFinalize = return Acid.initialAnalyticsPixelsState - } - } - - -- | Default constructor for building this feature. analyticsPixelsFeature :: ServerEnv - -> StateComponent AcidState Acid.AnalyticsPixelsState + -> Store.Backend -> CoreFeature -- To get site package list -> UserFeature -- To authenticate users -> UploadFeature -- For accessing package maintainers and trustees @@ -93,7 +75,7 @@ analyticsPixelsFeature :: ServerEnv -> AnalyticsPixelsFeature analyticsPixelsFeature ServerEnv{..} - analyticsPixelsState + Store.Backend{backendStore = analyticsPixelsState, backendState} CoreFeature { coreResource = CoreResource{..} } UserFeature{..} UploadFeature{..} @@ -104,7 +86,7 @@ analyticsPixelsFeature ServerEnv{..} analyticsPixelsFeatureInterface = (emptyHackageFeature "AnalyticsPixels") { featureDesc = "Allow users to attach analytics pixels to their packages", featureResources = [analyticsPixelsResource, userAnalyticsPixelsResource] - , featureState = [abstractAcidStateComponent analyticsPixelsState] + , featureState = backendState } analyticsPixelsResource :: Resource @@ -114,16 +96,16 @@ analyticsPixelsFeature ServerEnv{..} userAnalyticsPixelsResource = resourceAt "/user/:username/analytics-pixels.:format" getPackageAnalyticsPixels :: MonadIO m => PackageName -> m (Set AnalyticsPixel) - getPackageAnalyticsPixels name = - queryState analyticsPixelsState (Acid.AnalyticsPixelsForPackage name) + getPackageAnalyticsPixels = + Store.getPackageAnalyticsPixels analyticsPixelsState addPackageAnalyticsPixel :: MonadIO m => PackageName -> AnalyticsPixel -> m Bool addPackageAnalyticsPixel name pixel = do - added <- updateState analyticsPixelsState (Acid.AddPackageAnalyticsPixel name pixel) + added <- Store.addPackageAnalyticsPixel analyticsPixelsState name pixel when added $ runHook_ analyticsPixelAdded (name, pixel) pure added removePackageAnalyticsPixel :: MonadIO m => PackageName -> AnalyticsPixel -> m () removePackageAnalyticsPixel name pixel = do - updateState analyticsPixelsState (Acid.RemovePackageAnalyticsPixel name pixel) + Store.removePackageAnalyticsPixel analyticsPixelsState name pixel runHook_ analyticsPixelRemoved (name, pixel) diff --git a/src/Distribution/Server/Features/AnalyticsPixels/Acid.hs b/src/Distribution/Server/Features/AnalyticsPixels/Acid.hs new file mode 100644 index 000000000..ce6cd7a4b --- /dev/null +++ b/src/Distribution/Server/Features/AnalyticsPixels/Acid.hs @@ -0,0 +1,37 @@ +module Distribution.Server.Features.AnalyticsPixels.Acid + ( acidStore + ) where + +import qualified Distribution.Server.Features.AnalyticsPixels.State as Acid +import Distribution.Server.Features.AnalyticsPixels.Store +import Distribution.Server.Framework +import Distribution.Server.Framework.BackupRestore + +acidStore :: FilePath -> IO Backend +acidStore stateDir = do + analyticsPixelsState <- analyticsPixelsStateComponent stateDir + return Backend { + backendStore = Store { + getPackageAnalyticsPixels = \name -> queryState analyticsPixelsState (Acid.AnalyticsPixelsForPackage name) + , addPackageAnalyticsPixel = \name pixel -> updateState analyticsPixelsState (Acid.AddPackageAnalyticsPixel name pixel) + , removePackageAnalyticsPixel = \name pixel -> updateState analyticsPixelsState (Acid.RemovePackageAnalyticsPixel name pixel) + } + , backendState = [abstractAcidStateComponent analyticsPixelsState] + } + +-- | Define the backing store (i.e. database component) +analyticsPixelsStateComponent :: FilePath -> IO (StateComponent AcidState Acid.AnalyticsPixelsState) +analyticsPixelsStateComponent stateDir = do + st <- openLocalStateFrom (stateDir "db" "AnalyticsPixels") Acid.initialAnalyticsPixelsState + return StateComponent { + stateDesc = "Backing store for AnalyticsPixels feature" + , stateHandle = st + , getState = query st Acid.GetAnalyticsPixelsState + , putState = update st . Acid.ReplaceAnalyticsPixelsState + , resetState = analyticsPixelsStateComponent + , backupState = \_ _ -> [] + , restoreState = RestoreBackup { + restoreEntry = error "Unexpected backup entry" + , restoreFinalize = return Acid.initialAnalyticsPixelsState + } + } diff --git a/src/Distribution/Server/Features/AnalyticsPixels/Store.hs b/src/Distribution/Server/Features/AnalyticsPixels/Store.hs new file mode 100644 index 000000000..cd287269f --- /dev/null +++ b/src/Distribution/Server/Features/AnalyticsPixels/Store.hs @@ -0,0 +1,25 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.AnalyticsPixels.Store + ( Backend(..) + , Store(..) + ) where + +import Distribution.Server.Features.AnalyticsPixels.Types +import Distribution.Server.Framework (AbstractStateComponent) + +import Distribution.Package (PackageName) + +import Control.Monad.Trans (MonadIO) +import Data.Set (Set) + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + getPackageAnalyticsPixels :: forall m. MonadIO m => PackageName -> m (Set AnalyticsPixel) + , addPackageAnalyticsPixel :: forall m. MonadIO m => PackageName -> AnalyticsPixel -> m Bool + , removePackageAnalyticsPixel :: forall m. MonadIO m => PackageName -> AnalyticsPixel -> m () + } diff --git a/src/Distribution/Server/Features/Core.hs b/src/Distribution/Server/Features/Core.hs index 2bc6ff059..f30dc86dd 100644 --- a/src/Distribution/Server/Features/Core.hs +++ b/src/Distribution/Server/Features/Core.hs @@ -18,8 +18,6 @@ module Distribution.Server.Features.Core ( -- * Misc other utils packageExists, packageIdExists, - - packagesStateComponent, ) where -- stdlib @@ -37,9 +35,9 @@ import qualified Data.Vector as Vec -- hackage import Distribution.Server.Prelude -import Distribution.Server.Features.Core.Backup +import Distribution.Server.Features.Core.Acid (acidStore) +import qualified Distribution.Server.Features.Core.Store as Store import qualified Distribution.Server.Features.Core.State as Acid -import Distribution.Server.Features.Security.Migration import Distribution.Server.Features.Security.SHA256 (sha256) import Distribution.Server.Features.Users import Distribution.Server.Framework @@ -76,6 +74,15 @@ data CoreFeature = CoreFeature { -- | Retrieves the entire main package index. queryGetPackageIndex :: forall m. MonadIO m => m (PackageIndex PkgInfo), + -- | Retrieves all versions of a package. + queryLookupPackageName :: forall m. MonadIO m => PackageName -> m [PkgInfo], + + -- | Retrieves a specific package version. + queryLookupPackageId :: forall m. MonadIO m => PackageId -> m (Maybe PkgInfo), + + -- | Retrieves the latest version of every package. + queryLatestPackages :: forall m. MonadIO m => m [PkgInfo], + -- | Retrieve the raw tarball info queryGetIndexTarballInfo :: forall m. MonadIO m => m IndexTarballInfo, @@ -269,8 +276,8 @@ data CoreResource = CoreResource { initCoreFeature :: ServerEnv -> IO (UserFeature -> IO CoreFeature) initCoreFeature env@ServerEnv{serverStateDir, serverCacheDelay, serverVerbosity = verbosity} = do - -- Canonical state - packagesState <- packagesStateComponent verbosity False serverStateDir + packagesBackend <- acidStore env verbosity False serverStateDir + let packagesStore = Store.backendStore packagesBackend -- Hooks packageChangeHook <- newHook @@ -300,16 +307,16 @@ initCoreFeature env@ServerEnv{serverStateDir, serverCacheDelay, -- need any other kind of migration. migrateUpdateLog <- (isLeft . Acid.packageUpdateLog) <$> - queryState packagesState Acid.GetPackagesState + Store.getPackagesState packagesStore when migrateUpdateLog $ do -- Migrate Acid.PackagesState (introduce package update log) logTiming verbosity "migrating package update log" $ do userdb <- queryGetUserDb users - updateState packagesState (Acid.MigrateAddUpdateLog userdb) + Store.migrateAddUpdateLog packagesStore userdb -- Migrate PkgTarball logTiming verbosity "migrating PkgTarball" $ - migratePkgTarball_v1_to_v2 env packagesState + Store.migratePackageTarballs packagesStore -- Create a checkpoint -- @@ -325,11 +332,11 @@ initCoreFeature env@ServerEnv{serverStateDir, serverCacheDelay, -- reconstruct the package log rather than use the package log as it was -- constructed in the first place, and we might potentially lose -- information. - createCheckpoint (stateHandle packagesState) + Store.createStoreCheckpoint packagesStore rec let (feature, getIndexTarball) = coreFeature env users - packagesState indexTar + packagesBackend indexTar packageChangeHook preIndexUpdateHook packageDownloadHook @@ -354,29 +361,14 @@ initCoreFeature env@ServerEnv{serverStateDir, serverCacheDelay, PackageChangeAdd _ -> return () _ -> do additionalEntries <- concat <$> runHook preIndexUpdateHook packageChange - forM_ additionalEntries $ updateState packagesState . Acid.AddOtherIndexEntry + forM_ additionalEntries $ Store.addOtherIndexEntry packagesStore prodAsyncCache indexTar "package change" return feature -packagesStateComponent :: Verbosity -> Bool -> FilePath -> IO (StateComponent AcidState Acid.PackagesState) -packagesStateComponent verbosity freshDB stateDir = do - let stateFile = stateDir "db" "PackagesState" - st <- logTiming verbosity "Loaded PackagesState" $ - openLocalStateFrom stateFile (Acid.initialPackagesState freshDB) - return StateComponent { - stateDesc = "Main package database" - , stateHandle = st - , getState = query st Acid.GetPackagesState - , putState = update st . Acid.ReplacePackagesState - , backupState = \_ -> indexToAllVersions - , restoreState = packagesBackup - , resetState = packagesStateComponent verbosity True - } - coreFeature :: ServerEnv -> UserFeature - -> StateComponent AcidState Acid.PackagesState + -> Store.Backend -> AsyncCache IndexTarballInfo -> Hook PackageChange () -> Hook PackageChange [TarIndexEntry] @@ -385,7 +377,7 @@ coreFeature :: ServerEnv , IO IndexTarballInfo ) coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} - packagesState cacheIndexTarball + Store.Backend{backendStore = packagesStore, backendState} cacheIndexTarball packageChangeHook preIndexUpdateHook packageDownloadHook @@ -408,7 +400,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} , coreAdminDeauth , corePackUserDeauth ] - , featureState = [abstractAcidStateComponent packagesState] + , featureState = backendState , featureCaches = [ CacheComponent { cacheDesc = "main package index tarball", @@ -505,7 +497,16 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} -- Queries -- queryGetPackageIndex :: MonadIO m => m (PackageIndex PkgInfo) - queryGetPackageIndex = Acid.packageIndex <$> queryState packagesState Acid.GetPackagesState + queryGetPackageIndex = Acid.packageIndex <$> Store.getPackagesState packagesStore + + queryLookupPackageName :: MonadIO m => PackageName -> m [PkgInfo] + queryLookupPackageName = Store.lookupPackageName packagesStore + + queryLookupPackageId :: MonadIO m => PackageId -> m (Maybe PkgInfo) + queryLookupPackageId = Store.lookupPackageId packagesStore + + queryLatestPackages :: MonadIO m => m [PkgInfo] + queryLatestPackages = Store.latestPackages packagesStore queryGetIndexTarballInfo :: MonadIO m => m IndexTarballInfo queryGetIndexTarballInfo = readAsyncCache cacheIndexTarball @@ -526,12 +527,11 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} let pkginfo = Acid.mkPackageInfo pkgid cabalFile uploadinfo mtarball additionalEntries <- concat `liftM` runHook preIndexUpdateHook (PackageChangeAdd pkginfo) - successFlag <- updateState packagesState $ - Acid.AddPackage3 - pkginfo - uploadinfo - (userName userInfo) - additionalEntries + successFlag <- Store.addPackage packagesStore + pkginfo + uploadinfo + (userName userInfo) + additionalEntries loginfo maxBound ("updateState(AddPackage3," ++ display pkgid ++ ") -> " ++ show successFlag) if successFlag @@ -541,7 +541,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} updateDeletePackage :: MonadIO m => PackageId -> m Bool updateDeletePackage pkgid = logTiming maxBound ("updateDeletePackage " ++ display pkgid) $ do - mpkginfo <- updateState packagesState (Acid.DeletePackage pkgid) + mpkginfo <- Store.deletePackage packagesStore pkgid case mpkginfo of Nothing -> return False Just pkginfo -> do @@ -552,12 +552,11 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} updateAddPackageRevision pkgid cabalfile uploadinfo@(_, uid) = logTiming maxBound ("updateAddPackageRevision " ++ display pkgid) $ do usersdb <- queryGetUserDb let Just userInfo = lookupUserId uid usersdb - (moldpkginfo, newpkginfo) <- updateState packagesState $ - Acid.AddPackageRevision2 - pkgid - cabalfile - uploadinfo - (userName userInfo) + (moldpkginfo, newpkginfo) <- Store.addPackageRevision packagesStore + pkgid + cabalfile + uploadinfo + (userName userInfo) loginfo maxBound ("updateState(AddPackageRevision2," ++ display pkgid ++ ") -> " ++ maybe "Nothing" (const "Just _") moldpkginfo) case moldpkginfo of Nothing -> @@ -567,7 +566,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} updateAddPackageTarball :: MonadIO m => PackageId -> PkgTarball -> UploadInfo -> m Bool updateAddPackageTarball pkgid tarball uploadinfo = logTiming maxBound ("updateAddPackageTarball " ++ display pkgid) $ do - mpkginfo <- updateState packagesState (Acid.AddPackageTarball pkgid tarball uploadinfo) + mpkginfo <- Store.addPackageTarball packagesStore pkgid tarball uploadinfo case mpkginfo of Nothing -> return False @@ -576,7 +575,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} return True updateSetPackageUploader pkgid userid = do - mpkginfo <- updateState packagesState (Acid.SetPackageUploader pkgid userid) + mpkginfo <- Store.setPackageUploader packagesStore pkgid userid case mpkginfo of Nothing -> return False Just (oldpkginfo, newpkginfo) -> do @@ -584,7 +583,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} return True updateSetPackageUploadTime pkgid time = do - mpkginfo <- updateState packagesState (Acid.SetPackageUploadTime pkgid time) + mpkginfo <- Store.setPackageUploadTime packagesStore pkgid time case mpkginfo of Nothing -> return False Just (oldpkginfo, newpkginfo) -> do @@ -593,8 +592,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} updateArchiveIndexEntry :: MonadIO m => FilePath -> LazyByteString -> UTCTime -> m () updateArchiveIndexEntry entryName entryData entryTime = logTiming maxBound ("updateArchiveIndexEntry " ++ show entryName) $ do - updateState packagesState $ - Acid.AddOtherIndexEntry $ ExtraEntry entryName entryData entryTime + Store.addOtherIndexEntry packagesStore $ ExtraEntry entryName entryData entryTime runHook_ packageChangeHook (PackageChangeIndexExtra entryName entryData entryTime) -- Cache updates @@ -603,7 +601,7 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} getIndexTarball = do users <- queryGetUserDb -- note, changes here don't automatically propagate time <- getCurrentTime - Acid.PackagesState index (Right updateSeq) <- queryState packagesState Acid.GetPackagesState + Acid.PackagesState index (Right updateSeq) <- Store.getPackagesState packagesStore let updateLog = Foldable.toList updateSeq legacyTarball = Packages.Index.writeLegacy users @@ -638,8 +636,8 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} lookupPackageName :: PackageName -> ServerPartE [PkgInfo] lookupPackageName pkgname = do - pkgsIndex <- queryGetPackageIndex - case PackageIndex.lookupPackageName pkgsIndex pkgname of + pkgs <- queryLookupPackageName pkgname + case pkgs of [] -> packageError [MText "No such package in package index"] pkgs -> return pkgs @@ -649,8 +647,8 @@ coreFeature ServerEnv{serverBlobStore = store} UserFeature{..} -- pkgs is sorted by version number and non-empty return (last pkgs) lookupPackageId pkgid = do - pkgsIndex <- queryGetPackageIndex - case PackageIndex.lookupPackageId pkgsIndex pkgid of + mpkg <- queryLookupPackageId pkgid + case mpkg of Just pkg -> return pkg _ -> packageError [MText $ "No such package version for " ++ display (packageName pkgid)] diff --git a/src/Distribution/Server/Features/Core/Acid.hs b/src/Distribution/Server/Features/Core/Acid.hs new file mode 100644 index 000000000..9cbf31095 --- /dev/null +++ b/src/Distribution/Server/Features/Core/Acid.hs @@ -0,0 +1,65 @@ +module Distribution.Server.Features.Core.Acid + ( acidStore + , packagesStateComponent + ) where + +import Distribution.Server.Features.Core.Backup +import Distribution.Server.Features.Core.Store +import qualified Distribution.Server.Features.Core.State as Acid +import Distribution.Server.Features.Security.Migration +import Distribution.Server.Framework +import qualified Distribution.Server.Packages.PackageIndex as PackageIndex + +import qualified Data.List.NonEmpty as NE + +acidStore :: ServerEnv -> Verbosity -> Bool -> FilePath -> IO Backend +acidStore env verbosity freshDB stateDir = do + packagesState <- packagesStateComponent verbosity freshDB stateDir + pure Backend { + backendStore = Store { + getPackagesState = queryState packagesState Acid.GetPackagesState + , lookupPackageName = \pkgname -> do + packages <- queryState packagesState Acid.GetPackagesState + pure (PackageIndex.lookupPackageName (Acid.packageIndex packages) pkgname) + , lookupPackageId = \pkgid -> do + packages <- queryState packagesState Acid.GetPackagesState + pure (PackageIndex.lookupPackageId (Acid.packageIndex packages) pkgid) + , latestPackages = do + packages <- queryState packagesState Acid.GetPackagesState + pure (NE.last <$> PackageIndex.allPackagesByNameNE (Acid.packageIndex packages)) + , addPackage = \pkginfo uploadinfo username entries -> + updateState packagesState (Acid.AddPackage3 pkginfo uploadinfo username entries) + , deletePackage = \pkgid -> + updateState packagesState (Acid.DeletePackage pkgid) + , addPackageRevision = \pkgid cabalfile uploadinfo username -> + updateState packagesState (Acid.AddPackageRevision2 pkgid cabalfile uploadinfo username) + , addPackageTarball = \pkgid tarball uploadinfo -> + updateState packagesState (Acid.AddPackageTarball pkgid tarball uploadinfo) + , setPackageUploader = \pkgid userid -> + updateState packagesState (Acid.SetPackageUploader pkgid userid) + , setPackageUploadTime = \pkgid time -> + updateState packagesState (Acid.SetPackageUploadTime pkgid time) + , addOtherIndexEntry = \entry -> + updateState packagesState (Acid.AddOtherIndexEntry entry) + , migrateAddUpdateLog = \userdb -> + updateState packagesState (Acid.MigrateAddUpdateLog userdb) + , migratePackageTarballs = migratePkgTarball_v1_to_v2 env packagesState + , createStoreCheckpoint = createCheckpoint (stateHandle packagesState) + } + , backendState = [abstractAcidStateComponent packagesState] + } + +packagesStateComponent :: Verbosity -> Bool -> FilePath -> IO (StateComponent AcidState Acid.PackagesState) +packagesStateComponent verbosity freshDB stateDir = do + let stateFile = stateDir "db" "PackagesState" + st <- logTiming verbosity "Loaded PackagesState" $ + openLocalStateFrom stateFile (Acid.initialPackagesState freshDB) + return StateComponent { + stateDesc = "Main package database" + , stateHandle = st + , getState = query st Acid.GetPackagesState + , putState = update st . Acid.ReplacePackagesState + , backupState = \_ -> indexToAllVersions + , restoreState = packagesBackup + , resetState = packagesStateComponent verbosity True + } diff --git a/src/Distribution/Server/Features/Core/Store.hs b/src/Distribution/Server/Features/Core/Store.hs new file mode 100644 index 000000000..23856468a --- /dev/null +++ b/src/Distribution/Server/Features/Core/Store.hs @@ -0,0 +1,40 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.Core.Store + ( Backend(..) + , Store(..) + ) where + +import Distribution.Server.Framework (AbstractStateComponent) +import Distribution.Server.Features.Core.State (PackagesState) +import Distribution.Server.Packages.Index (TarIndexEntry) +import Distribution.Server.Packages.Types +import Distribution.Server.Users.Types (UserId, UserName) +import Distribution.Server.Users.Users (Users) + +import Distribution.Package (PackageId, PackageName) + +import Control.Monad.Trans (MonadIO) +import Data.Time.Clock (UTCTime) + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + getPackagesState :: forall m. MonadIO m => m PackagesState + , lookupPackageName :: forall m. MonadIO m => PackageName -> m [PkgInfo] + , lookupPackageId :: forall m. MonadIO m => PackageId -> m (Maybe PkgInfo) + , latestPackages :: forall m. MonadIO m => m [PkgInfo] + , addPackage :: forall m. MonadIO m => PkgInfo -> UploadInfo -> UserName -> [TarIndexEntry] -> m Bool + , deletePackage :: forall m. MonadIO m => PackageId -> m (Maybe PkgInfo) + , addPackageRevision :: forall m. MonadIO m => PackageId -> CabalFileText -> UploadInfo -> UserName -> m (Maybe PkgInfo, PkgInfo) + , addPackageTarball :: forall m. MonadIO m => PackageId -> PkgTarball -> UploadInfo -> m (Maybe (PkgInfo, PkgInfo)) + , setPackageUploader :: forall m. MonadIO m => PackageId -> UserId -> m (Maybe (PkgInfo, PkgInfo)) + , setPackageUploadTime :: forall m. MonadIO m => PackageId -> UTCTime -> m (Maybe (PkgInfo, PkgInfo)) + , addOtherIndexEntry :: forall m. MonadIO m => TarIndexEntry -> m () + , migrateAddUpdateLog :: forall m. MonadIO m => Users -> m () + , migratePackageTarballs :: IO () + , createStoreCheckpoint :: IO () + } diff --git a/src/Distribution/Server/Features/HaskellPlatform.hs b/src/Distribution/Server/Features/HaskellPlatform.hs index 90f5f241b..12cf7093d 100644 --- a/src/Distribution/Server/Features/HaskellPlatform.hs +++ b/src/Distribution/Server/Features/HaskellPlatform.hs @@ -6,17 +6,15 @@ module Distribution.Server.Features.HaskellPlatform ( ) where import Distribution.Server.Framework -import Distribution.Server.Framework.BackupRestore -import qualified Distribution.Server.Features.HaskellPlatform.State as Acid +import Distribution.Server.Features.HaskellPlatform.Acid (acidStore) +import qualified Distribution.Server.Features.HaskellPlatform.Store as Store import Distribution.Package import Distribution.Version import Distribution.Text import Data.Function -import qualified Data.Map as Map -import qualified Data.Set as Set -- Note: this can be generalized into dividing Hackage up into however many @@ -47,34 +45,15 @@ data PlatformResource = PlatformResource { initPlatformFeature :: ServerEnv -> IO (IO PlatformFeature) initPlatformFeature ServerEnv{serverStateDir} = do - platformState <- platformStateComponent serverStateDir + platformState <- acidStore serverStateDir return $ do let feature = platformFeature platformState return feature -platformStateComponent :: FilePath -> IO (StateComponent AcidState Acid.PlatformPackages) -platformStateComponent stateDir = do - st <- openLocalStateFrom (stateDir "db" "Acid.PlatformPackages") Acid.initialPlatformPackages - return StateComponent { - stateDesc = "Platform packages" - , stateHandle = st - , getState = query st Acid.GetPlatformPackages - , putState = update st . Acid.ReplacePlatformPackages - , resetState = platformStateComponent - -- TODO: backup - -- For now backup is just empty, as this package is basically featureless - -- It defines state, but there is no way at all to modify this state - , backupState = \_ _ -> [] - , restoreState = RestoreBackup { - restoreEntry = error "Unexpected backup entry for platform" - , restoreFinalize = return Acid.initialPlatformPackages - } - } - -platformFeature :: StateComponent AcidState Acid.PlatformPackages +platformFeature :: Store.Backend -> PlatformFeature -platformFeature platformState +platformFeature Store.Backend{..} = PlatformFeature{..} where platformFeatureInterface = (emptyHackageFeature "platform") { @@ -84,7 +63,7 @@ platformFeature platformState platformPackage , platformPackages ] - , featureState = [abstractAcidStateComponent platformState] + , featureState = backendState } platformResource = fix $ \r -> PlatformResource @@ -107,14 +86,13 @@ platformFeature platformState ------------------------------------------ -- functionality: showing status for a single package, and for all packages, adding a package, deleting a package platformVersions :: MonadIO m => PackageName -> m [Version] - platformVersions pkgname = liftM Set.toList $ queryState platformState $ Acid.GetPlatformPackage pkgname + platformVersions = Store.platformVersions backendStore platformPackageLatest :: MonadIO m => m [(PackageName, Version)] - platformPackageLatest = liftM (Map.toList . Map.map Set.findMax . Acid.blessedPackages) $ queryState platformState Acid.GetPlatformPackages + platformPackageLatest = Store.platformPackageLatest backendStore setPlatform :: MonadIO m => PackageName -> [Version] -> m () - setPlatform pkgname versions = updateState platformState $ Acid.SetPlatformPackage pkgname (Set.fromList versions) + setPlatform = Store.setPlatform backendStore removePlatform :: MonadIO m => PackageName -> m () - removePlatform pkgname = updateState platformState $ Acid.SetPlatformPackage pkgname Set.empty - + removePlatform = Store.removePlatform backendStore diff --git a/src/Distribution/Server/Features/HaskellPlatform/Acid.hs b/src/Distribution/Server/Features/HaskellPlatform/Acid.hs new file mode 100644 index 000000000..7ae06d481 --- /dev/null +++ b/src/Distribution/Server/Features/HaskellPlatform/Acid.hs @@ -0,0 +1,46 @@ +{-# LANGUAGE NamedFieldPuns #-} + +module Distribution.Server.Features.HaskellPlatform.Acid + ( acidStore + ) where + +import Distribution.Server.Framework +import Distribution.Server.Framework.BackupRestore + +import qualified Distribution.Server.Features.HaskellPlatform.State as Acid +import Distribution.Server.Features.HaskellPlatform.Store + +import qualified Data.Map as Map +import qualified Data.Set as Set + +acidStore :: FilePath -> IO Backend +acidStore stateDir = do + platformState <- platformStateComponent stateDir + pure Backend + { backendStore = Store + { platformVersions = \pkgname -> fmap Set.toList $ queryState platformState $ Acid.GetPlatformPackage pkgname + , platformPackageLatest = fmap (Map.toList . Map.map Set.findMax . Acid.blessedPackages) $ queryState platformState Acid.GetPlatformPackages + , setPlatform = \pkgname versions -> updateState platformState $ Acid.SetPlatformPackage pkgname (Set.fromList versions) + , removePlatform = \pkgname -> updateState platformState $ Acid.SetPlatformPackage pkgname Set.empty + } + , backendState = [abstractAcidStateComponent platformState] + } + +platformStateComponent :: FilePath -> IO (StateComponent AcidState Acid.PlatformPackages) +platformStateComponent stateDir = do + st <- openLocalStateFrom (stateDir "db" "Acid.PlatformPackages") Acid.initialPlatformPackages + return StateComponent { + stateDesc = "Platform packages" + , stateHandle = st + , getState = query st Acid.GetPlatformPackages + , putState = update st . Acid.ReplacePlatformPackages + , resetState = platformStateComponent + -- TODO: backup + -- For now backup is just empty, as this package is basically featureless + -- It defines state, but there is no way at all to modify this state + , backupState = \_ _ -> [] + , restoreState = RestoreBackup { + restoreEntry = error "Unexpected backup entry for platform" + , restoreFinalize = return Acid.initialPlatformPackages + } + } diff --git a/src/Distribution/Server/Features/HaskellPlatform/Store.hs b/src/Distribution/Server/Features/HaskellPlatform/Store.hs new file mode 100644 index 000000000..94c7a092c --- /dev/null +++ b/src/Distribution/Server/Features/HaskellPlatform/Store.hs @@ -0,0 +1,25 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.HaskellPlatform.Store + ( Backend(..) + , Store(..) + ) where + +import Distribution.Server.Framework (AbstractStateComponent) + +import Distribution.Package (PackageName) +import Distribution.Version (Version) + +import Control.Monad.Trans (MonadIO) + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + platformVersions :: forall m. MonadIO m => PackageName -> m [Version] + , platformPackageLatest :: forall m. MonadIO m => m [(PackageName, Version)] + , setPlatform :: forall m. MonadIO m => PackageName -> [Version] -> m () + , removePlatform :: forall m. MonadIO m => PackageName -> m () + } diff --git a/src/Distribution/Server/Features/Mirror.hs b/src/Distribution/Server/Features/Mirror.hs index 0db7660cd..253c6a63e 100644 --- a/src/Distribution/Server/Features/Mirror.hs +++ b/src/Distribution/Server/Features/Mirror.hs @@ -13,15 +13,14 @@ import Distribution.Server.Framework import Distribution.Server.Features.Core import Distribution.Server.Features.Users -import Distribution.Server.Users.State +import Distribution.Server.Features.Mirror.Acid (acidStore) +import qualified Distribution.Server.Features.Mirror.Store as Store import Distribution.Server.Packages.Types -import Distribution.Server.Users.Backup import Distribution.Server.Users.Types import Distribution.Server.Users.Users hiding (lookupUserName) import Distribution.Server.Users.Group (UserGroup(..), GroupDescription(..), nullDescription) import qualified Distribution.Server.Framework.BlobStorage as BlobStorage import qualified Distribution.Server.Packages.Unpack as Upload -import Distribution.Server.Framework.BackupDump import Distribution.Server.Util.Parse (unpackUTF8) import Distribution.PackageDescription.Parsec (parseGenericPackageDescription, runParseResult) @@ -61,7 +60,7 @@ initMirrorFeature :: ServerEnv -> IO MirrorFeature) initMirrorFeature env@ServerEnv{serverStateDir} = do -- Canonical state - mirrorersState <- mirrorersStateComponent serverStateDir + mirrorersState <- acidStore serverStateDir return $ \core user@UserFeature{..} -> do -- Tie the knot with a do-rec @@ -73,23 +72,10 @@ initMirrorFeature env@ServerEnv{serverStateDir} = do return feature -mirrorersStateComponent :: FilePath -> IO (StateComponent AcidState MirrorClients) -mirrorersStateComponent stateDir = do - st <- openLocalStateFrom (stateDir "db" "MirrorClients") initialMirrorClients - return StateComponent { - stateDesc = "Mirror clients" - , stateHandle = st - , getState = query st GetMirrorClients - , putState = update st . ReplaceMirrorClients . mirrorClients - , backupState = \_ (MirrorClients clients) -> [csvToBackup ["clients.csv"] $ groupToCSV clients] - , restoreState = MirrorClients <$> groupBackup ["clients.csv"] - , resetState = mirrorersStateComponent - } - mirrorFeature :: ServerEnv -> CoreFeature -> UserFeature - -> StateComponent AcidState MirrorClients + -> Store.Backend -> UserGroup -> GroupResource -> (MirrorFeature, UserGroup) @@ -106,7 +92,8 @@ mirrorFeature ServerEnv{serverBlobStore = store} , updateSetPackageUploader } UserFeature{..} - mirrorersState mirrorGroup mirrorGroupResource + Store.Backend{backendStore = mirrorersState, backendState} + mirrorGroup mirrorGroupResource = (MirrorFeature{..}, mirrorersGroupDesc) where mirrorFeatureInterface = (emptyHackageFeature "mirror") { @@ -121,7 +108,7 @@ mirrorFeature ServerEnv{serverBlobStore = store} [ groupResource mirrorGroupResource , groupUserResource mirrorGroupResource ] - , featureState = [abstractAcidStateComponent mirrorersState] + , featureState = backendState } mirrorResource = MirrorResource { @@ -152,9 +139,9 @@ mirrorFeature ServerEnv{serverBlobStore = store} mirrorersGroupDesc = UserGroup { groupDesc = nullDescription { groupTitle = "Mirror clients" }, - queryUserGroup = queryState mirrorersState GetMirrorClientsList, - addUserToGroup = updateState mirrorersState . AddMirrorClient, - removeUserFromGroup = updateState mirrorersState . RemoveMirrorClient, + queryUserGroup = Store.getMirrorClientsList mirrorersState, + addUserToGroup = Store.addMirrorClient mirrorersState, + removeUserFromGroup = Store.removeMirrorClient mirrorersState, groupsAllowedToDelete = [adminGroup], groupsAllowedToAdd = [adminGroup] } diff --git a/src/Distribution/Server/Features/Mirror/Acid.hs b/src/Distribution/Server/Features/Mirror/Acid.hs new file mode 100644 index 000000000..cfcad3f75 --- /dev/null +++ b/src/Distribution/Server/Features/Mirror/Acid.hs @@ -0,0 +1,45 @@ +module Distribution.Server.Features.Mirror.Acid + ( acidStore + ) where + +import Distribution.Server.Prelude + +import Distribution.Server.Framework +import Distribution.Server.Framework.BackupDump +import Distribution.Server.Features.Mirror.Store +import Distribution.Server.Users.State + ( MirrorClients(..) + , GetMirrorClients(..) + , GetMirrorClientsList(..) + , ReplaceMirrorClients(..) + , AddMirrorClient(..) + , RemoveMirrorClient(..) + , initialMirrorClients + , mirrorClients + ) +import Distribution.Server.Users.Backup (groupBackup, groupToCSV) + +acidStore :: FilePath -> IO Backend +acidStore stateDir = do + mirrorersState <- mirrorersStateComponent stateDir + pure Backend { + backendStore = Store { + getMirrorClientsList = queryState mirrorersState GetMirrorClientsList + , addMirrorClient = \uid -> updateState mirrorersState (AddMirrorClient uid) + , removeMirrorClient = \uid -> updateState mirrorersState (RemoveMirrorClient uid) + } + , backendState = [abstractAcidStateComponent mirrorersState] + } + +mirrorersStateComponent :: FilePath -> IO (StateComponent AcidState MirrorClients) +mirrorersStateComponent stateDir = do + st <- openLocalStateFrom (stateDir "db" "MirrorClients") initialMirrorClients + return StateComponent { + stateDesc = "Mirror clients" + , stateHandle = st + , getState = query st GetMirrorClients + , putState = update st . ReplaceMirrorClients . mirrorClients + , backupState = \_ (MirrorClients clients) -> [csvToBackup ["clients.csv"] $ groupToCSV clients] + , restoreState = MirrorClients <$> groupBackup ["clients.csv"] + , resetState = mirrorersStateComponent + } diff --git a/src/Distribution/Server/Features/Mirror/Store.hs b/src/Distribution/Server/Features/Mirror/Store.hs new file mode 100644 index 000000000..93b338691 --- /dev/null +++ b/src/Distribution/Server/Features/Mirror/Store.hs @@ -0,0 +1,23 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.Mirror.Store + ( Backend(..) + , Store(..) + ) where + +import Distribution.Server.Framework (AbstractStateComponent) +import Distribution.Server.Users.UserIdSet (UserIdSet) +import Distribution.Server.Users.Types (UserId) + +import Control.Monad.Trans (MonadIO) + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + getMirrorClientsList :: forall m. MonadIO m => m UserIdSet + , addMirrorClient :: forall m. MonadIO m => UserId -> m () + , removeMirrorClient :: forall m. MonadIO m => UserId -> m () + } diff --git a/src/Distribution/Server/Features/PackageList.hs b/src/Distribution/Server/Features/PackageList.hs index d2b063c29..5f03931cd 100644 --- a/src/Distribution/Server/Features/PackageList.hs +++ b/src/Distribution/Server/Features/PackageList.hs @@ -19,7 +19,6 @@ import Distribution.Server.Users.Users (userIdToName) import qualified Distribution.Server.Users.UserIdSet as UserIdSet import Distribution.Server.Users.Group(UserGroup(..), GroupDescription(..)) import Distribution.Server.Features.PreferredVersions -import qualified Distribution.Server.Packages.PackageIndex as PackageIndex import Distribution.Server.Util.CountingMap (cmFind) import Distribution.Server.Packages.Types @@ -152,8 +151,8 @@ initListFeature _env = do registerHookJust packageChangeHook isPackageAdd $ \pkg -> do let pkgname = packageName . packageId $ pkg prefsinfo <- queryGetPreferredInfo pkgname - index <- queryGetPackageIndex - let allVersions = packageVersion <$> PackageIndex.lookupPackageName index pkgname + pkgs <- queryLookupPackageName pkgname + let allVersions = packageVersion <$> pkgs modifyItem pkgname $ \x -> updateReferenceVersion prefsinfo allVersions $ x @@ -196,8 +195,8 @@ initListFeature _env = do runHook_ itemUpdate (Set.singleton pkgname) registerHook updatePreferredHook $ \(pkgname, prefsinfo) -> do - index <- queryGetPackageIndex - let allVersions = packageVersion <$> PackageIndex.lookupPackageName index pkgname + pkgs <- queryLookupPackageName pkgname + let allVersions = packageVersion <$> pkgs modifyItem pkgname $ updateReferenceVersion prefsinfo allVersions return feature @@ -252,15 +251,13 @@ listFeature CoreFeature{..} case hasItem of True -> modifyMemState itemCache $ Map.adjust token pkgname False -> do - index <- queryGetPackageIndex - let pkgs = PackageIndex.lookupPackageName index pkgname + pkgs <- queryLookupPackageName pkgname case pkgs of [] -> return () --this shouldn't happen _ -> modifyMemState itemCache . uncurry Map.insert =<< constructItem (last pkgs) updateDesc pkgname = do - index <- queryGetPackageIndex - let pkgs = PackageIndex.lookupPackageName index pkgname + pkgs <- queryLookupPackageName pkgname case pkgs of [] -> modifyMemState itemCache (Map.delete pkgname) _ -> modifyItem pkgname (updateDescriptionItem $ pkgDesc $ last pkgs) @@ -276,8 +273,8 @@ listFeature CoreFeature{..} constructItemIndex :: IO (Map PackageName PackageItem) constructItemIndex = do - index <- queryGetPackageIndex - items <- mapM (constructItem . last) $ PackageIndex.allPackagesByName index + latestPackages <- queryLatestPackages + items <- mapM constructItem latestPackages return $ Map.fromList items constructItem :: PkgInfo -> IO (PackageName, PackageItem) diff --git a/src/Distribution/Server/Features/Search.hs b/src/Distribution/Server/Features/Search.hs index 9bae0e2f3..6d57f9513 100644 --- a/src/Distribution/Server/Features/Search.hs +++ b/src/Distribution/Server/Features/Search.hs @@ -12,7 +12,6 @@ import Distribution.Server.Features.PackageList import Distribution.Server.Features.Search.PkgSearch import qualified Distribution.Server.Features.Search.SearchEngine as SearchEngine -import qualified Distribution.Server.Packages.PackageIndex as PackageIndex import Distribution.Server.Packages.Types @@ -102,12 +101,11 @@ searchFeature ServerEnv{serverBaseURI} CoreFeature{..} ListFeature{getAllLists} getSearchDoc = flattenPackageDescription . pkgDesc postInit = do - pkgindex <- queryGetPackageIndex + latestPackages <- queryLatestPackages pkgdownloads <- getDownloadCounts let pkgs = [ (getSearchDoc pkgLatestVer, pkgdownloads pkgname) - | pkgVers <- PackageIndex.allPackagesByName pkgindex - , let pkgLatestVer = last pkgVers - pkgname = packageName pkgLatestVer ] + | pkgLatestVer <- latestPackages + , let pkgname = packageName pkgLatestVer ] se = SearchEngine.insertDocs pkgs initialPkgSearchEngine writeMemState searchEngineState se @@ -117,8 +115,7 @@ searchFeature ServerEnv{serverBaseURI} CoreFeature{..} ListFeature{getAllLists} --TODO: update periodically for download count changes updatePackage :: PackageName -> IO () updatePackage pkgname = do - index <- queryGetPackageIndex - let pkgs = PackageIndex.lookupPackageName index pkgname + pkgs <- queryLookupPackageName pkgname case reverse pkgs of [] -> modifyMemState searchEngineState (SearchEngine.deleteDoc pkgname) diff --git a/src/Distribution/Server/Features/Tags.hs b/src/Distribution/Server/Features/Tags.hs index 09b4723e8..ae51e24a4 100644 --- a/src/Distribution/Server/Features/Tags.hs +++ b/src/Distribution/Server/Features/Tags.hs @@ -154,7 +154,7 @@ tagsFeature :: CoreFeature -> MemState (Map PackageName (Set Tag, Set Tag)) -> TagsFeature -tagsFeature CoreFeature{ queryGetPackageIndex } +tagsFeature CoreFeature{ queryLatestPackages } UploadFeature{ maintainersGroup, trusteesGroup } UserFeature{ guardAuthorised' } tagsState @@ -200,8 +200,8 @@ tagsFeature CoreFeature{ queryGetPackageIndex } initImmutableTags :: IO () initImmutableTags = do - index <- queryGetPackageIndex - let calcTags = Acid.tagPackages $ constructImmutableTagIndex index + latestPackages <- queryLatestPackages + let calcTags = Acid.tagPackages $ constructImmutableTagIndex latestPackages aliases <- mapM (queryState tagsAlias . Acid.GetTagAlias) $ Map.keys calcTags let calcTags' = Map.toList . Map.fromListWith Set.union $ zip aliases (Map.elems calcTags) forM_ calcTags' $ uncurry setCalculatedTag @@ -240,14 +240,15 @@ tagsFeature CoreFeature{ queryGetPackageIndex } mergeTags targetTag deprTag = case simpleParse =<< targetTag of Just (Tag orig) -> do - index <- queryGetPackageIndex + latestPkgs <- queryLatestPackages + let pkgNames = packageName <$> latestPkgs void $ updateState tagsAlias $ Acid.AddTagAlias (Tag orig) deprTag - void $ constructMergedTagIndex (Tag orig) deprTag index + void $ constructMergedTagIndex (Tag orig) deprTag pkgNames _ -> errBadRequest "Tag not recognised" [MText "Couldn't parse tag. It should be a single tag."] -- tags on merging - constructMergedTagIndex :: forall m. (Functor m, MonadIO m) => Tag -> Tag -> PackageIndex PkgInfo -> m Acid.PackageTags - constructMergedTagIndex orig depr = foldM addToTags Acid.emptyPackageTags . PackageIndex.allPackageNames + constructMergedTagIndex :: forall m. (Functor m, MonadIO m) => Tag -> Tag -> [PackageName] -> m Acid.PackageTags + constructMergedTagIndex orig depr = foldM addToTags Acid.emptyPackageTags where addToTags calcTags pn = do pkgTags <- queryTagsForPackage pn if Set.member depr pkgTags @@ -312,10 +313,10 @@ constructTagIndex = foldl' addToTags Acid.emptyPackageTags . PackageIndex.allPac in Acid.setTags pkgname (Set.union categoryTags immutableTags) pkgTags -- tags on startup -constructImmutableTagIndex :: PackageIndex PkgInfo -> Acid.PackageTags -constructImmutableTagIndex = foldl' addToTags Acid.emptyPackageTags . PackageIndex.allPackagesByName - where addToTags calcTags pkgList = - let info = pkgDesc $ last pkgList +constructImmutableTagIndex :: [PkgInfo] -> Acid.PackageTags +constructImmutableTagIndex = foldl' addToTags Acid.emptyPackageTags + where addToTags calcTags pkg = + let info = pkgDesc pkg !pn = packageName info !tags = constructImmutableTags info in Acid.setTags pn (Set.fromList tags) calcTags diff --git a/src/Distribution/Server/Features/TarIndexCache.hs b/src/Distribution/Server/Features/TarIndexCache.hs index e075d1b8c..83bbda3a0 100644 --- a/src/Distribution/Server/Features/TarIndexCache.hs +++ b/src/Distribution/Server/Features/TarIndexCache.hs @@ -17,6 +17,8 @@ import Distribution.Server.Framework import Distribution.Server.Framework.BlobStorage import qualified Distribution.Server.Framework.BlobStorage as BlobStorage import Distribution.Server.Framework.BackupRestore +import Distribution.Server.Features.TarIndexCache.Acid (acidStore) +import qualified Distribution.Server.Features.TarIndexCache.Store as Store import qualified Distribution.Server.Features.TarIndexCache.State as Acid import Distribution.Server.Features.Users import Distribution.Server.Packages.Types @@ -45,36 +47,19 @@ initTarIndexCacheFeature :: ServerEnv -> IO (UserFeature -> IO TarIndexCacheFeature) initTarIndexCacheFeature env@ServerEnv{serverStateDir} = do - tarIndexCache <- tarIndexCacheStateComponent serverStateDir + tarIndexCacheBackend <- acidStore serverStateDir return $ \users -> do - let feature = tarIndexCacheFeature env users tarIndexCache + let feature = tarIndexCacheFeature env users tarIndexCacheBackend return feature -tarIndexCacheStateComponent :: FilePath -> IO (StateComponent AcidState Acid.TarIndexCache) -tarIndexCacheStateComponent stateDir = do - st <- openLocalStateFrom (stateDir "db" "TarIndexCache") Acid.initialTarIndexCache - return StateComponent { - stateDesc = "Mapping from tarball blob IDs to tarindex blob IDs" - , stateHandle = st - , getState = query st Acid.GetTarIndexCache - , putState = update st . Acid.ReplaceTarIndexCache - , resetState = tarIndexCacheStateComponent - -- We don't backup the tar indices, but reconstruct them on demand - , backupState = \_ _ -> [] - , restoreState = RestoreBackup { - restoreEntry = error "The impossible happened" - , restoreFinalize = return Acid.initialTarIndexCache - } - } - tarIndexCacheFeature :: ServerEnv -> UserFeature - -> StateComponent AcidState Acid.TarIndexCache + -> Store.Backend -> TarIndexCacheFeature tarIndexCacheFeature ServerEnv{serverBlobStore = store} UserFeature{..} - tarIndexCache = + Store.Backend{backendStore = tarIndexCache, backendState} = TarIndexCacheFeature{..} where tarIndexCacheFeatureInterface :: HackageFeature @@ -84,7 +69,7 @@ tarIndexCacheFeature ServerEnv{serverBlobStore = store} -- (TODO: We could potentially check that if a package occurs in both -- packages then both caches point to identical tar indices, but for -- that we would need to be in IO) - , featureState = [abstractAcidStateComponent' (\_ _ -> []) tarIndexCache] + , featureState = backendState , featureResources = [ (resourceAt "/server-status/tarindices.:format") { resourceDesc = [ (GET, "Which tar indices have been generated?") @@ -99,7 +84,7 @@ tarIndexCacheFeature ServerEnv{serverBlobStore = store} -- This is the heart of this feature cachedTarIndex :: BlobId -> IO TarIndex cachedTarIndex tarBallBlobId = do - mTarIndexBlobId <- queryState tarIndexCache (Acid.FindTarIndex tarBallBlobId) + mTarIndexBlobId <- Store.findTarIndex tarIndexCache tarBallBlobId case mTarIndexBlobId of Just tarIndexBlobId -> do serializedTarIndex <- fetch store tarIndexBlobId @@ -112,7 +97,7 @@ tarIndexCacheFeature ServerEnv{serverBlobStore = store} Left err -> throwIO (userError err) Right tarIndex -> return tarIndex tarIndexBlobId <- add store (runPutLazy (safePut tarIndex)) - updateState tarIndexCache (Acid.SetTarIndex tarBallBlobId tarIndexBlobId) + Store.setTarIndex tarIndexCache tarBallBlobId tarIndexBlobId return tarIndex cachedPackageTarIndex :: PkgTarball -> IO TarIndex @@ -120,7 +105,7 @@ tarIndexCacheFeature ServerEnv{serverBlobStore = store} serveTarIndicesStatus :: ServerPartE Response serveTarIndicesStatus = do - Acid.TarIndexCache state <- liftIO $ getState tarIndexCache + Acid.TarIndexCache state <- liftIO $ Store.getTarIndexCache tarIndexCache return . toResponse . toJSON . Map.toList $ state -- | With curl: @@ -131,7 +116,7 @@ tarIndexCacheFeature ServerEnv{serverBlobStore = store} guardAuthorised_ [InGroup adminGroup] -- TODO: This resets the tar indices _state_ only, we don't actually -- remove any blobs - liftIO $ putState tarIndexCache Acid.initialTarIndexCache + liftIO $ Store.replaceTarIndexCache tarIndexCache Acid.initialTarIndexCache ok $ toResponse "Ok!" -- Functions to access specific files in a tarball diff --git a/src/Distribution/Server/Features/TarIndexCache/Acid.hs b/src/Distribution/Server/Features/TarIndexCache/Acid.hs new file mode 100644 index 000000000..75b58d49f --- /dev/null +++ b/src/Distribution/Server/Features/TarIndexCache/Acid.hs @@ -0,0 +1,41 @@ +module Distribution.Server.Features.TarIndexCache.Acid + ( acidStore + ) where + +import Distribution.Server.Prelude + +import Distribution.Server.Framework +import Distribution.Server.Framework.BackupRestore +import Distribution.Server.Features.TarIndexCache.Store +import Distribution.Server.Features.TarIndexCache.State as Acid + +acidStore :: FilePath -> IO Backend +acidStore stateDir = do + st <- openLocalStateFrom (stateDir "db" "TarIndexCache") Acid.initialTarIndexCache + state <- tarIndexCacheStateComponent stateDir + pure Backend { + backendStore = Store { + getTarIndexCache = query st Acid.GetTarIndexCache + , replaceTarIndexCache = update st . Acid.ReplaceTarIndexCache + , findTarIndex = query st . Acid.FindTarIndex + , setTarIndex = \tar index -> update st (Acid.SetTarIndex tar index) + } + , backendState = [abstractAcidStateComponent' (\_ _ -> []) state] + } + +tarIndexCacheStateComponent :: FilePath -> IO (StateComponent AcidState Acid.TarIndexCache) +tarIndexCacheStateComponent stateDir = do + st <- openLocalStateFrom (stateDir "db" "TarIndexCache") Acid.initialTarIndexCache + return StateComponent { + stateDesc = "Mapping from tarball blob IDs to tarindex blob IDs" + , stateHandle = st + , getState = query st Acid.GetTarIndexCache + , putState = update st . Acid.ReplaceTarIndexCache + , resetState = tarIndexCacheStateComponent + -- We don't backup the tar indices, but reconstruct them on demand + , backupState = \_ _ -> [] + , restoreState = RestoreBackup { + restoreEntry = error "The impossible happened" + , restoreFinalize = return Acid.initialTarIndexCache + } + } diff --git a/src/Distribution/Server/Features/TarIndexCache/Store.hs b/src/Distribution/Server/Features/TarIndexCache/Store.hs new file mode 100644 index 000000000..7f618ef1a --- /dev/null +++ b/src/Distribution/Server/Features/TarIndexCache/Store.hs @@ -0,0 +1,22 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.TarIndexCache.Store + ( Backend(..) + , Store(..) + ) where + +import Distribution.Server.Framework (AbstractStateComponent) +import Distribution.Server.Features.TarIndexCache.State (TarIndexCache) +import Distribution.Server.Framework.BlobStorage (BlobId) + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + getTarIndexCache :: IO TarIndexCache + , replaceTarIndexCache :: TarIndexCache -> IO () + , findTarIndex :: BlobId -> IO (Maybe BlobId) + , setTarIndex :: BlobId -> BlobId -> IO () + } diff --git a/src/Distribution/Server/Features/Votes.hs b/src/Distribution/Server/Features/Votes.hs index 8739824c1..b23761e8c 100644 --- a/src/Distribution/Server/Features/Votes.hs +++ b/src/Distribution/Server/Features/Votes.hs @@ -4,15 +4,21 @@ -- module Distribution.Server.Features.Votes ( VotesFeature(..) + , Backend(..) + , Store(..) , initVotesFeature + , initVotesFeatureWith ) where import Distribution.Server.Features.Votes.Types (Score) -import qualified Distribution.Server.Features.Votes.State as Acid +import Distribution.Server.Features.Votes.Acid (acidStore) import qualified Distribution.Server.Features.Votes.Render as Render +import Distribution.Server.Features.Votes.Store + ( votesScore + , Backend(..) + , Store(..) ) import Distribution.Server.Framework -import Distribution.Server.Framework.BackupRestore import Distribution.Server.Features.Core import Distribution.Server.Features.Users @@ -52,7 +58,15 @@ initVotesFeature :: ServerEnv -> UserFeature -> IO VotesFeature) initVotesFeature env@ServerEnv{serverStateDir} = do - dbVotesState <- votesStateComponent serverStateDir + initVotesFeatureWith (acidStore serverStateDir) env + +initVotesFeatureWith :: IO Backend + -> ServerEnv + -> IO ( CoreFeature + -> UserFeature + -> IO VotesFeature ) +initVotesFeatureWith openVotesStore env = do + dbVotesState <- openVotesStore updateVotes <- newHook return $ \coref@CoreFeature{..} userf@UserFeature{..} -> do @@ -62,34 +76,16 @@ initVotesFeature env@ServerEnv{serverStateDir} = do return feature --- | Define the backing store (i.e. database component) -votesStateComponent :: FilePath -> IO (StateComponent AcidState Acid.VotesState) -votesStateComponent stateDir = do - st <- openLocalStateFrom (stateDir "db" "Votes") Acid.initialVotesState - return StateComponent { - stateDesc = "Backing store for Map PackageName -> Users who voted for it" - , stateHandle = st - , getState = query st Acid.GetVotesState - , putState = update st . Acid.ReplaceVotesState - , resetState = votesStateComponent - , backupState = \_ _ -> [] - , restoreState = RestoreBackup { - restoreEntry = error "Unexpected backup entry" - , restoreFinalize = return $ Acid.VotesState Map.empty - } - } - - -- | Default constructor for building this feature. votesFeature :: ServerEnv - -> StateComponent AcidState Acid.VotesState + -> Backend -> CoreFeature -- To get site package list -> UserFeature -- To authenticate users -> Hook (PackageName, Float) () -> VotesFeature votesFeature ServerEnv{..} - votesState + Backend{backendStore = votesState, backendState} CoreFeature { coreResource = CoreResource{..} } UserFeature{..} votesUpdated @@ -100,7 +96,7 @@ votesFeature ServerEnv{..} featureResources = [ packagesVotesResource , packageVotesResource ] - , featureState = [abstractAcidStateComponent votesState] + , featureState = backendState } @@ -129,9 +125,9 @@ votesFeature ServerEnv{..} servePackageVotesGet :: DynamicPath -> ServerPartE Response servePackageVotesGet _ = do cacheControlWithoutETag [Public, maxAgeMinutes 10] - votesMap <- queryState votesState Acid.GetAllPackageVoteSets + votesMap <- getAllPackageVoteSets votesState ok . toResponse $ objectL - [ (display pkgname, toJSON (Acid.votesScore pkgMap)) + [ (display pkgname, toJSON (votesScore pkgMap)) | (pkgname, pkgMap) <- Map.toList votesMap ] -- Get the number of votes a package has. If the package @@ -161,7 +157,7 @@ votesFeature ServerEnv{..} "2" -> pure 2 "3" -> pure 3 _ -> fail "invalid score value received" - _ <- updateState votesState (Acid.AddVote pkgname uid score) + _ <- addVote votesState pkgname uid score pkgScore <- pkgNumScore pkgname runHook_ votesUpdated (pkgname, pkgScore) ok . toResponse $ "Package voted for successfully" @@ -174,7 +170,7 @@ votesFeature ServerEnv{..} pkgname <- packageInPath dpath guardValidPackageName pkgname - success <- updateState votesState (Acid.RemoveVote pkgname uid) + success <- removeVote votesState pkgname uid pkgScore <- pkgNumScore pkgname when success $ runHook_ votesUpdated (pkgname, pkgScore) @@ -187,21 +183,17 @@ votesFeature ServerEnv{..} -- Returns true if a user has previously voted for the -- package in question. didUserVote :: MonadIO m => PackageName -> UserId -> m Bool - didUserVote pkgname uid = - queryState votesState (Acid.GetPackageUserVoted pkgname uid) + didUserVote = getPackageUserVoted votesState -- Returns the number of votes a package has. pkgNumVotes :: MonadIO m => PackageName -> m Int - pkgNumVotes pkgname = - queryState votesState (Acid.GetPackageVoteCount pkgname) + pkgNumVotes = getPackageVoteCount votesState pkgNumScore :: MonadIO m => PackageName -> m Float - pkgNumScore pkgname = - queryState votesState (Acid.GetPackageVoteScore pkgname) + pkgNumScore = getPackageVoteScore votesState pkgUserVote :: MonadIO m => PackageName -> UserId -> m (Maybe Score) - pkgUserVote pkgname uid = - queryState votesState (Acid.GetPackageUserVote pkgname uid) + pkgUserVote = getPackageUserVote votesState -- Renders the HTML for the "Votes:" section on package pages. renderVotesHtml :: PackageName -> ServerPartE X.Html diff --git a/src/Distribution/Server/Features/Votes/Acid.hs b/src/Distribution/Server/Features/Votes/Acid.hs new file mode 100644 index 000000000..aa9ee0997 --- /dev/null +++ b/src/Distribution/Server/Features/Votes/Acid.hs @@ -0,0 +1,45 @@ +module Distribution.Server.Features.Votes.Acid + ( acidStore + , votesStateComponent + ) where + +import Distribution.Server.Features.Votes.Store +import qualified Distribution.Server.Features.Votes.State as Acid + +import Distribution.Server.Framework +import Distribution.Server.Framework.BackupRestore + +import qualified Data.Map as Map + +acidStore :: FilePath -> IO Backend +acidStore stateDir = do + votesState <- votesStateComponent stateDir + return Backend { + backendStore = Store { + getAllPackageVoteSets = queryState votesState Acid.GetAllPackageVoteSets + , addVote = \pkgname uid score -> updateState votesState (Acid.AddVote pkgname uid score) + , removeVote = \pkgname uid -> updateState votesState (Acid.RemoveVote pkgname uid) + , getPackageVoteCount = \pkgname -> queryState votesState (Acid.GetPackageVoteCount pkgname) + , getPackageVoteScore = \pkgname -> queryState votesState (Acid.GetPackageVoteScore pkgname) + , getPackageUserVoted = \pkgname uid -> queryState votesState (Acid.GetPackageUserVoted pkgname uid) + , getPackageUserVote = \pkgname uid -> queryState votesState (Acid.GetPackageUserVote pkgname uid) + } + , backendState = [abstractAcidStateComponent votesState] + } + +-- | Define the backing store (i.e. database component) +votesStateComponent :: FilePath -> IO (StateComponent AcidState Acid.VotesState) +votesStateComponent stateDir = do + st <- openLocalStateFrom (stateDir "db" "Votes") Acid.initialVotesState + return StateComponent { + stateDesc = "Backing store for Map PackageName -> Users who voted for it" + , stateHandle = st + , getState = query st Acid.GetVotesState + , putState = update st . Acid.ReplaceVotesState + , resetState = votesStateComponent + , backupState = \_ _ -> [] + , restoreState = RestoreBackup { + restoreEntry = error "Unexpected backup entry" + , restoreFinalize = return $ Acid.VotesState Map.empty + } + } diff --git a/src/Distribution/Server/Features/Votes/State.hs b/src/Distribution/Server/Features/Votes/State.hs index 17b05bda3..3cb718d7e 100644 --- a/src/Distribution/Server/Features/Votes/State.hs +++ b/src/Distribution/Server/Features/Votes/State.hs @@ -4,6 +4,7 @@ module Distribution.Server.Features.Votes.State where import Distribution.Server.Features.Votes.Types +import Distribution.Server.Features.Votes.Store (votesScore) import Distribution.Server.Framework.MemSize import Distribution.Package (PackageName) @@ -15,9 +16,7 @@ import Distribution.Server.Users.State () import Data.Map (Map) import qualified Data.Map as Map -import Data.List import Data.Maybe (fromMaybe) -import Control.Arrow ((&&&)) import Data.Acid (Query, Update, makeAcidic) import Data.SafeCopy (base, extension, deriveSafeCopy, Migrate(..)) @@ -55,16 +54,6 @@ userVotedForPackage pkgname uid votes = Nothing -> False Just _ -> True --- Using a Bayesian average (m=1.5, C=2) to calculate scoring -votesScore :: Map UserId Score -> Float -votesScore m = - let grouping = map (head &&& length) . group . sort . Map.elems $ m - score :: Float - score = fromIntegral ((sum $ map (uncurry (*)) grouping) + 3)/ - fromIntegral (2 + sum (map snd grouping)) - roundedScore = fromIntegral (round (score * 4) :: Int) / 4 - in roundedScore - -- All the acid state transactions addVote :: PackageName -> UserId -> Score -> Update VotesState Float diff --git a/src/Distribution/Server/Features/Votes/Store.hs b/src/Distribution/Server/Features/Votes/Store.hs new file mode 100644 index 000000000..5cc87fd61 --- /dev/null +++ b/src/Distribution/Server/Features/Votes/Store.hs @@ -0,0 +1,44 @@ +{-# LANGUAGE RankNTypes #-} + +module Distribution.Server.Features.Votes.Store + ( Backend(..) + , Store(..) + , votesScore + ) where + +import Distribution.Server.Features.Votes.Types +import Distribution.Server.Framework.Feature (AbstractStateComponent) +import Distribution.Server.Users.Types (UserId) + +import Distribution.Package (PackageName) + +import Control.Arrow ((&&&)) +import Control.Monad.Trans (MonadIO) +import Data.List (group, sort) +import Data.Map (Map) +import qualified Data.Map as Map + +data Backend = Backend { + backendStore :: Store + , backendState :: [AbstractStateComponent] + } + +data Store = Store { + getAllPackageVoteSets :: forall m. MonadIO m => m (Map.Map PackageName (Map.Map UserId Score)) + , addVote :: forall m. MonadIO m => PackageName -> UserId -> Score -> m Float + , removeVote :: forall m. MonadIO m => PackageName -> UserId -> m Bool + , getPackageVoteCount :: forall m. MonadIO m => PackageName -> m Int + , getPackageVoteScore :: forall m. MonadIO m => PackageName -> m Float + , getPackageUserVoted :: forall m. MonadIO m => PackageName -> UserId -> m Bool + , getPackageUserVote :: forall m. MonadIO m => PackageName -> UserId -> m (Maybe Score) + } + +-- Using a Bayesian average (m=1.5, C=2) to calculate scoring +votesScore :: Map UserId Score -> Float +votesScore m = + let grouping = map (head &&& length) . group . sort . Map.elems $ m + score :: Float + score = fromIntegral ((sum $ map (uncurry (*)) grouping) + 3)/ + fromIntegral (2 + sum (map snd grouping)) + roundedScore = fromIntegral (round (score * 4) :: Int) / 4 + in roundedScore