using System.Diagnostics; using Minio; using Minio.DataModel.Args; using Npgsql; using skyscraper5.Data; using skyscraper5.Data.PostgreSql; using skyscraper5.Skyscraper.Plugins; using skyscraper5.Skyscraper.Scraper.Storage; using skyscraper5.Skyscraper.Scraper.Storage.Split; namespace skyscraper5.Storage.PostgresqlMinio { [SkyscraperPlugin] [ScrapeStorageFactoryId(3,"PostgreSQL & MinIO",false)] public class PostgresqlMinioStorageFactory : IScraperStorageFactory { public IScraperStroage CreateScraperStroage() { NpgsqlConnectionStringBuilder ncsb = new NpgsqlConnectionStringBuilder(); ncsb.Database = PostgreSqlDatabase; ncsb.ApplicationName = "skyscraper5"; ncsb.Host = PostgreSqlHost; ncsb.Password = PostgreSqlPassword; ncsb.Pooling = true; ncsb.Port = PostgreSqlPort; ncsb.Username = PostgreSqlUsername; if (Debugger.IsAttached) ncsb.Timeout = 1024; IMinioClient mc = new MinioClient() .WithEndpoint(MinioEndpoint) .WithCredentials(MinioAccessKey, MinioSecretKey) .WithSSL(MinioSecure).Build(); BucketExistsArgs bucketExistsArgs = new BucketExistsArgs().WithBucket(MinioBucket); bool bucketExists = mc.BucketExistsAsync(bucketExistsArgs).Result; if (!bucketExists) { MakeBucketArgs makeBucketArgs = new MakeBucketArgs().WithBucket(MinioBucket); mc.MakeBucketAsync(makeBucketArgs).Wait(); } PostgresqlDataStore postgresDs = new PostgresqlDataStore(ncsb); MinioObjectStorage minioOs = new MinioObjectStorage(mc, MinioBucket); return new SplitScraperStorage(postgresDs, minioOs); } public string MinioBucket { get; set; } public bool MinioSecure { get; set; } public string MinioSecretKey { get; set; } public string MinioAccessKey { get; set; } public string MinioEndpoint { get; set; } public string PostgreSqlUsername { get; set; } public ushort PostgreSqlPort { get; set; } public string PostgreSqlPassword { get; set; } public string PostgreSqlHost { get; set; } public string PostgreSqlDatabase { get; set; } } }